From 953d537ba36d2be8321a8b2a21ccc56c53882004 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 28 Sep 2021 10:02:37 +0200 Subject: [PATCH 01/14] Remove default maxmind geoip databases from distribution and adjusted tests. Kept the `geolite2-databases` dependency for tests only. Relates to #68920 --- modules/ingest-geoip/build.gradle | 7 -- ...gDatabasesWhilePerformingGeoLookupsIT.java | 11 +- .../ingest/geoip/DatabaseRegistry.java | 6 +- .../ingest/geoip/GeoIpProcessor.java | 17 ++- .../ingest/geoip/LocalDatabases.java | 64 +---------- .../ingest/geoip/DatabaseRegistryTests.java | 19 ++-- .../geoip/GeoIpProcessorFactoryTests.java | 106 +++++++----------- .../ingest/geoip/LocalDatabasesTests.java | 69 +++++------- 8 files changed, 107 insertions(+), 192 deletions(-) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index b2d8689e5c2e6..1c2cf0b4fc49d 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -57,14 +57,7 @@ tasks.named("internalClusterTest").configure { } } -tasks.register("copyDefaultGeoIp2DatabaseFiles", Copy) { - from { zipTree(configurations.testCompileClasspath.files.find { it.name.contains('geolite2-databases') }) } - into "${project.buildDir}/ingest-geoip" - include "*.mmdb" -} - tasks.named("bundlePlugin").configure { - dependsOn("copyDefaultGeoIp2DatabaseFiles") from("${project.buildDir}/ingest-geoip") { into '/' } diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java index afa40c241fd64..290662738b61d 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java @@ -56,10 +56,9 @@ public class ReloadingDatabasesWhilePerformingGeoLookupsIT extends ESTestCase { * geoip processor instance is using the related {@link DatabaseReaderLazyLoader} instance */ public void test() throws Exception { - Path geoIpModulesDir = createTempDir(); Path geoIpConfigDir = createTempDir(); Path geoIpTmpDir = createTempDir(); - DatabaseRegistry databaseRegistry = createRegistry(geoIpModulesDir, geoIpConfigDir, geoIpTmpDir); + DatabaseRegistry databaseRegistry = createRegistry(geoIpConfigDir, geoIpTmpDir); ClusterService clusterService = mock(ClusterService.class); when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry, clusterService); @@ -164,13 +163,13 @@ public void test() throws Exception { assertThat(lazyLoader.current(), equalTo(0)); } // Avoid accumulating many temp dirs while running with -Dtests.iters=X - IOUtils.rm(geoIpModulesDir, geoIpConfigDir, geoIpTmpDir); + IOUtils.rm(geoIpConfigDir, geoIpTmpDir); } - private static DatabaseRegistry createRegistry(Path geoIpModulesDir, Path geoIpConfigDir, Path geoIpTmpDir) throws IOException { - copyDatabaseFiles(geoIpModulesDir); + private static DatabaseRegistry createRegistry(Path geoIpConfigDir, Path geoIpTmpDir) throws IOException { GeoIpCache cache = new GeoIpCache(0); - LocalDatabases localDatabases = new LocalDatabases(geoIpModulesDir, geoIpConfigDir, cache); + LocalDatabases localDatabases = new LocalDatabases(geoIpConfigDir, cache); + copyDatabaseFiles(geoIpConfigDir, localDatabases); DatabaseRegistry databaseRegistry = new DatabaseRegistry(geoIpTmpDir, mock(Client.class), cache, localDatabases, Runnable::run); databaseRegistry.initialize("nodeId", mock(ResourceWatcherService.class), mock(IngestService.class)); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java index d6ecd3ffced1e..7562058c7983c 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java @@ -152,12 +152,12 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { ingestService.addIngestClusterStateListener(this::checkDatabases); } - public DatabaseReaderLazyLoader getDatabase(String name, boolean fallbackUsingDefaultDatabases) { + public DatabaseReaderLazyLoader getDatabase(String name) { // There is a need for reference counting in order to avoid using an instance // that gets closed while using it. (this can happen during a database update) while (true) { DatabaseReaderLazyLoader instance = - databases.getOrDefault(name, localDatabases.getDatabase(name, fallbackUsingDefaultDatabases)); + databases.getOrDefault(name, localDatabases.getDatabase(name)); if (instance == null || instance.preLookup()) { return instance; } @@ -167,7 +167,7 @@ public DatabaseReaderLazyLoader getDatabase(String name, boolean fallbackUsingDe } List getAllDatabases() { - List all = new ArrayList<>(localDatabases.getAllDatabases()); + List all = new ArrayList<>(localDatabases.getConfigDatabases().values()); this.databases.forEach((key, value) -> all.add(value)); return all; } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 5ca1efe361b9f..7480fcdd5b2b0 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -23,6 +23,8 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkAddress; @@ -52,6 +54,10 @@ public final class GeoIpProcessor extends AbstractProcessor { + private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(GeoIpProcessor.class); + static final String DEFAULT_DATABASES_DEPRECATION_MESSAGE = "the [fallback_to_default_databases] has been deprecated," + + " because Elasticsearch no longer includes the default Maxmind geoip databases. This setting will be removed in Elasticsearch 9.0"; + public static final String TYPE = "geoip"; private static final String CITY_DB_SUFFIX = "-City"; private static final String COUNTRY_DB_SUFFIX = "-Country"; @@ -379,9 +385,14 @@ public GeoIpProcessor create( List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); boolean firstOnly = readBooleanProperty(TYPE, processorTag, config, "first_only", true); - boolean fallbackUsingDefaultDatabases = readBooleanProperty(TYPE, processorTag, config, "fallback_to_default_databases", true); - DatabaseReaderLazyLoader lazyLoader = databaseRegistry.getDatabase(databaseFile, fallbackUsingDefaultDatabases); + // noop, should be removed in 9.0 + Object value = config.remove("fallback_to_default_databases"); + if (value != null) { + DEPRECATION_LOGGER.critical(DeprecationCategory.OTHER, "default_databases_message", DEFAULT_DATABASES_DEPRECATION_MESSAGE); + } + + DatabaseReaderLazyLoader lazyLoader = databaseRegistry.getDatabase(databaseFile); if (lazyLoader == null) { throw newConfigurationException(TYPE, processorTag, "database_file", "database file [" + databaseFile + "] doesn't exist"); @@ -417,7 +428,7 @@ public GeoIpProcessor create( } } CheckedSupplier supplier = () -> { - DatabaseReaderLazyLoader loader = databaseRegistry.getDatabase(databaseFile, fallbackUsingDefaultDatabases); + DatabaseReaderLazyLoader loader = databaseRegistry.getDatabase(databaseFile); if (loader == null) { throw new ResourceNotFoundException("database file [" + databaseFile + "] doesn't exist"); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java index 99b557a6de3be..4f15b40c32557 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java @@ -11,8 +11,6 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; -import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.PathUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.watcher.FileChangesListener; import org.elasticsearch.watcher.FileWatcher; @@ -23,22 +21,17 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Stream; -import static org.elasticsearch.ingest.geoip.IngestGeoIpPlugin.DEFAULT_DATABASE_FILENAMES; - /** * Keeps track of the databases locally available to a node: - * 1) Default databases shipped with the default distribution via ingest-geoip module - * 2) User provided databases from the ES_HOME/config/ingest-geoip directory. This directory is monitored + * 1) User provided databases from the ES_HOME/config/ingest-geoip directory. This directory is monitored * and files updates are picked up and may cause databases being loaded or removed at runtime. */ final class LocalDatabases implements Closeable { @@ -48,28 +41,16 @@ final class LocalDatabases implements Closeable { private final GeoIpCache cache; private final Path geoipConfigDir; - private final Map defaultDatabases; private final ConcurrentMap configDatabases; LocalDatabases(Environment environment, GeoIpCache cache) { - this( - // In GeoIpProcessorNonIngestNodeTests, ingest-geoip is loaded on the classpath. - // This means that the plugin is never unbundled into a directory where the database files would live. - // Therefore, we have to copy these database files ourselves. To do this, we need the ability to specify where - // those database files would go. We do this by adding a plugin that registers ingest.geoip.database_path as an - // actual setting. Otherwise, in production code, this setting is not registered and the database path is not configurable. - environment.settings().get("ingest.geoip.database_path") != null ? - getGeoipConfigDirectory(environment) : - environment.modulesFile().resolve("ingest-geoip"), - environment.configFile().resolve("ingest-geoip"), - cache); + this(environment.configFile().resolve("ingest-geoip"), cache); } - LocalDatabases(Path geoipModuleDir, Path geoipConfigDir, GeoIpCache cache) { + LocalDatabases(Path geoipConfigDir, GeoIpCache cache) { this.cache = cache; this.geoipConfigDir = geoipConfigDir; this.configDatabases = new ConcurrentHashMap<>(); - this.defaultDatabases = initDefaultDatabases(geoipModuleDir); } void initialize(ResourceWatcherService resourceWatcher) throws IOException { @@ -79,22 +60,11 @@ void initialize(ResourceWatcherService resourceWatcher) throws IOException { watcher.addListener(new GeoipDirectoryListener()); resourceWatcher.add(watcher, ResourceWatcherService.Frequency.HIGH); - LOGGER.info("initialized default databases [{}], config databases [{}] and watching [{}] for changes", - defaultDatabases.keySet(), configDatabases.keySet(), geoipConfigDir); - } - - DatabaseReaderLazyLoader getDatabase(String name, boolean fallbackUsingDefaultDatabases) { - return configDatabases.getOrDefault(name, fallbackUsingDefaultDatabases ? defaultDatabases.get(name) : null); - } - - List getAllDatabases() { - List all = new ArrayList<>(defaultDatabases.values()); - all.addAll(configDatabases.values()); - return all; + LOGGER.info("initialized config databases [{}] and watching [{}] for changes", configDatabases.keySet(), geoipConfigDir); } - Map getDefaultDatabases() { - return defaultDatabases; + DatabaseReaderLazyLoader getDatabase(String name) { + return configDatabases.get(name); } Map getConfigDatabases() { @@ -122,20 +92,6 @@ void updateDatabase(Path file, boolean update) { } } - Map initDefaultDatabases(Path geoipModuleDir) { - Map databases = new HashMap<>(DEFAULT_DATABASE_FILENAMES.length); - - for (String filename : DEFAULT_DATABASE_FILENAMES) { - Path source = geoipModuleDir.resolve(filename); - assert Files.exists(source); - String databaseFileName = source.getFileName().toString(); - DatabaseReaderLazyLoader loader = new DatabaseReaderLazyLoader(cache, source, null); - databases.put(databaseFileName, loader); - } - - return Collections.unmodifiableMap(databases); - } - Map initConfigDatabases(Path geoipConfigDir) throws IOException { Map databases = new HashMap<>(); @@ -161,19 +117,11 @@ Map initConfigDatabases(Path geoipConfigDir) t @Override public void close() throws IOException { - for (DatabaseReaderLazyLoader lazyLoader : defaultDatabases.values()) { - lazyLoader.close(); - } for (DatabaseReaderLazyLoader lazyLoader : configDatabases.values()) { lazyLoader.close(); } } - @SuppressForbidden(reason = "PathUtils#get") - private static Path getGeoipConfigDirectory(Environment environment) { - return PathUtils.get(environment.settings().get("ingest.geoip.database_path")); - } - private class GeoipDirectoryListener implements FileChangesListener { @Override diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java index 73803f5b78d19..e4548eb672684 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java @@ -100,18 +100,17 @@ public class DatabaseRegistryTests extends ESTestCase { @Before public void setup() throws IOException { - final Path geoIpDir = createTempDir(); final Path geoIpConfigDir = createTempDir(); Files.createDirectories(geoIpConfigDir); - copyDatabaseFiles(geoIpDir); + GeoIpCache cache = new GeoIpCache(1000); + LocalDatabases localDatabases = new LocalDatabases(geoIpConfigDir, cache); + copyDatabaseFiles(geoIpConfigDir, localDatabases); threadPool = new TestThreadPool(LocalDatabases.class.getSimpleName()); Settings settings = Settings.builder().put("resource.reload.interval.high", TimeValue.timeValueMillis(100)).build(); resourceWatcherService = new ResourceWatcherService(settings, threadPool); client = mock(Client.class); - GeoIpCache cache = new GeoIpCache(1000); - LocalDatabases localDatabases = new LocalDatabases(geoIpDir, geoIpConfigDir, cache); geoIpTmpDir = createTempDir(); databaseRegistry = new DatabaseRegistry(geoIpTmpDir, client, cache, localDatabases, Runnable::run); databaseRegistry.initialize("nodeId", resourceWatcherService, mock(IngestService.class)); @@ -139,9 +138,9 @@ public void testCheckDatabases() throws Exception { .routingTable(createIndexRoutingTable()) .build(); - assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb", false), nullValue()); + assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb"), nullValue()); databaseRegistry.checkDatabases(state); - DatabaseReaderLazyLoader database = databaseRegistry.getDatabase("GeoIP2-City.mmdb", false); + DatabaseReaderLazyLoader database = databaseRegistry.getDatabase("GeoIP2-City.mmdb"); assertThat(database, nullValue()); verify(client, times(0)).search(any()); try (Stream files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) { @@ -160,7 +159,7 @@ public void testCheckDatabases() throws Exception { .routingTable(createIndexRoutingTable()) .build(); databaseRegistry.checkDatabases(state); - database = databaseRegistry.getDatabase("GeoIP2-City.mmdb", false); + database = databaseRegistry.getDatabase("GeoIP2-City.mmdb"); assertThat(database, notNullValue()); verify(client, times(10)).search(any()); //30 days check passed but we mocked mmdb data so parsing will fail @@ -184,7 +183,7 @@ public void testCheckDatabases_dontCheckDatabaseOnNonIngestNode() throws Excepti .build(); databaseRegistry.checkDatabases(state); - assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb", false), nullValue()); + assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb"), nullValue()); verify(client, never()).search(any()); try (Stream files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) { assertThat(files.collect(Collectors.toList()), empty()); @@ -206,7 +205,7 @@ public void testCheckDatabases_dontCheckDatabaseWhenNoDatabasesIndex() throws Ex .build(); databaseRegistry.checkDatabases(state); - assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb", false), nullValue()); + assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb"), nullValue()); verify(client, never()).search(any()); try (Stream files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) { assertThat(files.collect(Collectors.toList()), empty()); @@ -227,7 +226,7 @@ public void testCheckDatabases_dontCheckDatabaseWhenGeoIpDownloadTask() throws E mockSearches("GeoIP2-City.mmdb", 0, 9); databaseRegistry.checkDatabases(state); - assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb", false), nullValue()); + assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb"), nullValue()); verify(client, never()).search(any()); try (Stream files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) { assertThat(files.collect(Collectors.toList()), empty()); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index f47de667692f4..36d036cfaacfb 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -45,7 +46,6 @@ import java.util.Set; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -53,20 +53,21 @@ public class GeoIpProcessorFactoryTests extends ESTestCase { private Path geoipTmpDir; + private Path geoIpConfigDir; + private LocalDatabases localDatabases; private DatabaseRegistry databaseRegistry; private ClusterService clusterService; @Before public void loadDatabaseReaders() throws IOException { - final Path geoIpDir = createTempDir(); final Path configDir = createTempDir(); - final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); + geoIpConfigDir = configDir.resolve("ingest-geoip"); Files.createDirectories(geoIpConfigDir); - copyDatabaseFiles(geoIpDir); Client client = mock(Client.class); GeoIpCache cache = new GeoIpCache(1000); - LocalDatabases localDatabases = new LocalDatabases(geoIpDir, geoIpConfigDir, new GeoIpCache(1000)); + localDatabases = new LocalDatabases(geoIpConfigDir, new GeoIpCache(1000)); + copyDatabaseFiles(geoIpConfigDir, localDatabases); geoipTmpDir = createTempDir(); databaseRegistry = new DatabaseRegistry(geoipTmpDir, client, cache, localDatabases, Runnable::run); clusterService = mock(ClusterService.class); @@ -252,21 +253,20 @@ public void testBuildIllegalFieldOption() throws Exception { } public void testLazyLoading() throws Exception { - final Path geoIpDir = createTempDir(); final Path configDir = createTempDir(); final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); Files.createDirectories(geoIpConfigDir); - copyDatabaseFiles(geoIpDir); + GeoIpCache cache = new GeoIpCache(1000); + LocalDatabases localDatabases = new LocalDatabases(geoIpConfigDir, cache); + copyDatabaseFiles(geoIpConfigDir, localDatabases); // Loading another database reader instances, because otherwise we can't test lazy loading as the // database readers used at class level are reused between tests. (we want to keep that otherwise running this // test will take roughly 4 times more time) Client client = mock(Client.class); - GeoIpCache cache = new GeoIpCache(1000); - LocalDatabases localDatabases = new LocalDatabases(geoIpDir, geoIpConfigDir, cache); DatabaseRegistry databaseRegistry = new DatabaseRegistry(createTempDir(), client, cache, localDatabases, Runnable::run); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry, clusterService); - for (DatabaseReaderLazyLoader lazyLoader : localDatabases.getAllDatabases()) { + for (DatabaseReaderLazyLoader lazyLoader : localDatabases.getConfigDatabases().values()) { assertNull(lazyLoader.databaseReader.get()); } @@ -279,10 +279,10 @@ public void testLazyLoading() throws Exception { final GeoIpProcessor city = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here - assertNull(databaseRegistry.getDatabase("GeoLite2-City.mmdb", true).databaseReader.get()); + assertNull(databaseRegistry.getDatabase("GeoLite2-City.mmdb").databaseReader.get()); city.execute(document); // the first ingest should trigger a database load - assertNotNull(databaseRegistry.getDatabase("GeoLite2-City.mmdb", true).databaseReader.get()); + assertNotNull(databaseRegistry.getDatabase("GeoLite2-City.mmdb").databaseReader.get()); config = new HashMap<>(); config.put("field", "_field"); @@ -290,10 +290,10 @@ public void testLazyLoading() throws Exception { final GeoIpProcessor country = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here - assertNull(databaseRegistry.getDatabase("GeoLite2-Country.mmdb", true).databaseReader.get()); + assertNull(databaseRegistry.getDatabase("GeoLite2-Country.mmdb").databaseReader.get()); country.execute(document); // the first ingest should trigger a database load - assertNotNull(databaseRegistry.getDatabase("GeoLite2-Country.mmdb", true).databaseReader.get()); + assertNotNull(databaseRegistry.getDatabase("GeoLite2-Country.mmdb").databaseReader.get()); config = new HashMap<>(); config.put("field", "_field"); @@ -301,18 +301,18 @@ public void testLazyLoading() throws Exception { final GeoIpProcessor asn = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here - assertNull(databaseRegistry.getDatabase("GeoLite2-ASN.mmdb", true).databaseReader.get()); + assertNull(databaseRegistry.getDatabase("GeoLite2-ASN.mmdb").databaseReader.get()); asn.execute(document); // the first ingest should trigger a database load - assertNotNull(databaseRegistry.getDatabase("GeoLite2-ASN.mmdb", true).databaseReader.get()); + assertNotNull(databaseRegistry.getDatabase("GeoLite2-ASN.mmdb").databaseReader.get()); } public void testLoadingCustomDatabase() throws IOException { - final Path geoIpDir = createTempDir(); final Path configDir = createTempDir(); final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); Files.createDirectories(geoIpConfigDir); - copyDatabaseFiles(geoIpDir); + LocalDatabases localDatabases = new LocalDatabases(geoIpConfigDir, new GeoIpCache(1000)); + copyDatabaseFiles(geoIpConfigDir, localDatabases); // fake the GeoIP2-City database copyDatabaseFile(geoIpConfigDir, "GeoLite2-City.mmdb"); Files.move(geoIpConfigDir.resolve("GeoLite2-City.mmdb"), geoIpConfigDir.resolve("GeoIP2-City.mmdb")); @@ -323,13 +323,12 @@ public void testLoadingCustomDatabase() throws IOException { */ ThreadPool threadPool = new TestThreadPool("test"); ResourceWatcherService resourceWatcherService = new ResourceWatcherService(Settings.EMPTY, threadPool); - LocalDatabases localDatabases = new LocalDatabases(geoIpDir, geoIpConfigDir, new GeoIpCache(1000)); Client client = mock(Client.class); GeoIpCache cache = new GeoIpCache(1000); DatabaseRegistry databaseRegistry = new DatabaseRegistry(createTempDir(), client, cache, localDatabases, Runnable::run); databaseRegistry.initialize("nodeId", resourceWatcherService, mock(IngestService.class)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry, clusterService); - for (DatabaseReaderLazyLoader lazyLoader : localDatabases.getAllDatabases()) { + for (DatabaseReaderLazyLoader lazyLoader : localDatabases.getConfigDatabases().values()) { assertNull(lazyLoader.databaseReader.get()); } @@ -342,32 +341,21 @@ public void testLoadingCustomDatabase() throws IOException { final GeoIpProcessor city = factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here - assertNull(databaseRegistry.getDatabase("GeoIP2-City.mmdb", true).databaseReader.get()); + assertNull(databaseRegistry.getDatabase("GeoIP2-City.mmdb").databaseReader.get()); city.execute(document); // the first ingest should trigger a database load - assertNotNull(databaseRegistry.getDatabase("GeoIP2-City.mmdb", true).databaseReader.get()); + assertNotNull(databaseRegistry.getDatabase("GeoIP2-City.mmdb").databaseReader.get()); resourceWatcherService.close(); threadPool.shutdown(); } public void testFallbackUsingDefaultDatabases() throws Exception { GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry, clusterService); - { - Map config = new HashMap<>(); - config.put("field", "source_field"); - config.put("fallback_to_default_databases", false); - Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config)); - assertThat(e.getMessage(), equalTo("[database_file] database file [GeoLite2-City.mmdb] doesn't exist")); - } - { - Map config = new HashMap<>(); - config.put("field", "source_field"); - if (randomBoolean()) { - config.put("fallback_to_default_databases", true); - } - GeoIpProcessor processor = factory.create(null, null, null, config); - assertThat(processor, notNullValue()); - } + Map config = new HashMap<>(); + config.put("field", "source_field"); + config.put("fallback_to_default_databases", randomBoolean()); + factory.create(null, null, null, config); + assertWarnings(GeoIpProcessor.DEFAULT_DATABASES_DEPRECATION_MESSAGE); } public void testDefaultDatabaseWithTaskPresent() throws Exception { @@ -390,43 +378,31 @@ public void testDefaultDatabaseWithTaskPresent() throws Exception { processor.execute(RandomDocumentPicks.randomIngestDocument(random(), Map.of("_field", "89.160.20.128"))); } - public void testFallbackUsingDefaultDatabasesWhileIngesting() throws Exception { - copyDatabaseFile(geoipTmpDir, "GeoLite2-City-Test.mmdb"); + public void testUpdateDatabaseWhileIngesting() throws Exception { GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry, clusterService); - // fallback_to_default_databases=true, first use default city db then a custom city db: + Map config = new HashMap<>(); + config.put("field", "source_field"); + GeoIpProcessor processor = factory.create(null, null, null, config); + Map document = new HashMap<>(); + document.put("source_field", "89.160.20.128"); { - Map config = new HashMap<>(); - config.put("field", "source_field"); - if (randomBoolean()) { - config.put("fallback_to_default_databases", true); - } - GeoIpProcessor processor = factory.create(null, null, null, config); - Map document = new HashMap<>(); - document.put("source_field", "89.160.20.128"); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); processor.execute(ingestDocument); Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("geoip"); assertThat(geoData.get("city_name"), equalTo("Tumba")); - - databaseRegistry.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City-Test.mmdb")); - ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - processor.execute(ingestDocument); - geoData = (Map) ingestDocument.getSourceAndMetadata().get("geoip"); - assertThat(geoData.get("city_name"), equalTo("Linköping")); } - // fallback_to_default_databases=false, first use a custom city db then remove the custom db and expect failure: { - Map config = new HashMap<>(); - config.put("field", "source_field"); - config.put("fallback_to_default_databases", false); - GeoIpProcessor processor = factory.create(null, null, null, config); - Map document = new HashMap<>(); - document.put("source_field", "89.160.20.128"); + copyDatabaseFile(geoipTmpDir, "GeoLite2-City-Test.mmdb"); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + databaseRegistry.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City-Test.mmdb")); processor.execute(ingestDocument); Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("geoip"); assertThat(geoData.get("city_name"), equalTo("Linköping")); + } + { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); databaseRegistry.removeStaleEntries(List.of("GeoLite2-City.mmdb")); + localDatabases.updateDatabase(geoIpConfigDir.resolve("GeoLite2-City.mmdb"), false); Exception e = expectThrows(ResourceNotFoundException.class, () -> processor.execute(ingestDocument)); assertThat(e.getMessage(), equalTo("database file [GeoLite2-City.mmdb] doesn't exist")); } @@ -435,13 +411,15 @@ public void testFallbackUsingDefaultDatabasesWhileIngesting() throws Exception { private static void copyDatabaseFile(final Path path, final String databaseFilename) throws IOException { Files.copy( new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/" + databaseFilename)), - path.resolve(databaseFilename) + path.resolve(databaseFilename), + StandardCopyOption.REPLACE_EXISTING ); } - static void copyDatabaseFiles(final Path path) throws IOException { + static void copyDatabaseFiles(final Path path, LocalDatabases localDatabases) throws IOException { for (final String databaseFilename : IngestGeoIpPlugin.DEFAULT_DATABASE_FILENAMES) { copyDatabaseFile(path, databaseFilename); + localDatabases.updateDatabase(path.resolve(databaseFilename), true); } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/LocalDatabasesTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/LocalDatabasesTests.java index 2c2d4e73faeb2..4d27b22e7db19 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/LocalDatabasesTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/LocalDatabasesTests.java @@ -20,11 +20,14 @@ import org.junit.Before; import java.io.IOException; +import java.nio.file.CopyOption; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; public class LocalDatabasesTests extends ESTestCase { @@ -46,19 +49,13 @@ public void cleanup() { public void testLocalDatabasesEmptyConfig() throws Exception { Path configDir = createTempDir(); - LocalDatabases localDatabases = new LocalDatabases(prepareModuleDir(), configDir, new GeoIpCache(0)); + LocalDatabases localDatabases = new LocalDatabases(configDir, new GeoIpCache(0)); localDatabases.initialize(resourceWatcherService); - assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); - assertThat(localDatabases.getConfigDatabases().size(), equalTo(0)); - DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-ASN.mmdb", true); - assertThat(loader.getDatabaseType(), equalTo("GeoLite2-ASN")); - - loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); - assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - - loader = localDatabases.getDatabase("GeoLite2-Country.mmdb", true); - assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country")); + assertThat(localDatabases.getConfigDatabases(), anEmptyMap()); + assertThat(localDatabases.getDatabase("GeoLite2-ASN.mmdb"), nullValue()); + assertThat(localDatabases.getDatabase("GeoLite2-City.mmdb"), nullValue()); + assertThat(localDatabases.getDatabase("GeoLite2-Country.mmdb"), nullValue()); } public void testDatabasesConfigDir() throws Exception { @@ -66,55 +63,48 @@ public void testDatabasesConfigDir() throws Exception { Files.copy(LocalDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb")); Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb")); - LocalDatabases localDatabases = new LocalDatabases(prepareModuleDir(), configDir, new GeoIpCache(0)); + LocalDatabases localDatabases = new LocalDatabases(configDir, new GeoIpCache(0)); localDatabases.initialize(resourceWatcherService); - assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); assertThat(localDatabases.getConfigDatabases().size(), equalTo(2)); - DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-ASN.mmdb", true); - assertThat(loader.getDatabaseType(), equalTo("GeoLite2-ASN")); - - loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - loader = localDatabases.getDatabase("GeoLite2-Country.mmdb", true); - assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country")); - - loader = localDatabases.getDatabase("GeoIP2-City.mmdb", true); + loader = localDatabases.getDatabase("GeoIP2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoIP2-City")); } public void testDatabasesDynamicUpdateConfigDir() throws Exception { - Path configDir = createTempDir(); - LocalDatabases localDatabases = new LocalDatabases(prepareModuleDir(), configDir, new GeoIpCache(0)); + Path configDir = prepareConfigDir(); + LocalDatabases localDatabases = new LocalDatabases(configDir, new GeoIpCache(0)); localDatabases.initialize(resourceWatcherService); { - assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); - DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-ASN.mmdb", true); + assertThat(localDatabases.getConfigDatabases().size(), equalTo(3)); + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-ASN.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-ASN")); - loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + loader = localDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - loader = localDatabases.getDatabase("GeoLite2-Country.mmdb", true); + loader = localDatabases.getDatabase("GeoLite2-Country.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country")); } + CopyOption option = StandardCopyOption.REPLACE_EXISTING; Files.copy(LocalDatabases.class.getResourceAsStream("/GeoIP2-City-Test.mmdb"), configDir.resolve("GeoIP2-City.mmdb")); - Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb")); + Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"), option); assertBusy(() -> { - assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); - assertThat(localDatabases.getConfigDatabases().size(), equalTo(2)); - DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-ASN.mmdb", true); + assertThat(localDatabases.getConfigDatabases().size(), equalTo(4)); + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-ASN.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-ASN")); - loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + loader = localDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - loader = localDatabases.getDatabase("GeoLite2-Country.mmdb", true); + loader = localDatabases.getDatabase("GeoLite2-Country.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-Country")); - loader = localDatabases.getDatabase("GeoIP2-City.mmdb", true); + loader = localDatabases.getDatabase("GeoIP2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoIP2-City")); }); } @@ -123,14 +113,13 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { Path configDir = createTempDir(); Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), configDir.resolve("GeoLite2-City.mmdb")); GeoIpCache cache = new GeoIpCache(1000); // real cache to test purging of entries upon a reload - LocalDatabases localDatabases = new LocalDatabases(prepareModuleDir(), configDir, cache); + LocalDatabases localDatabases = new LocalDatabases(configDir, cache); localDatabases.initialize(resourceWatcherService); { assertThat(cache.count(), equalTo(0)); - assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); assertThat(localDatabases.getConfigDatabases().size(), equalTo(1)); - DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); CityResponse cityResponse = loader.getCity(InetAddresses.forString("89.160.20.128")); assertThat(cityResponse.getCity().getName(), equalTo("Tumba")); @@ -140,11 +129,10 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), configDir.resolve("GeoLite2-City.mmdb"), StandardCopyOption.REPLACE_EXISTING); assertBusy(() -> { - assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); assertThat(localDatabases.getConfigDatabases().size(), equalTo(1)); assertThat(cache.count(), equalTo(0)); - DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-City.mmdb", true); + DatabaseReaderLazyLoader loader = localDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); CityResponse cityResponse = loader.getCity(InetAddresses.forString("89.160.20.128")); assertThat(cityResponse.getCity().getName(), equalTo("Linköping")); @@ -153,13 +141,12 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { Files.delete(configDir.resolve("GeoLite2-City.mmdb")); assertBusy(() -> { - assertThat(localDatabases.getDefaultDatabases().size(), equalTo(3)); assertThat(localDatabases.getConfigDatabases().size(), equalTo(0)); assertThat(cache.count(), equalTo(0)); }); } - private static Path prepareModuleDir() throws IOException { + private static Path prepareConfigDir() throws IOException { Path dir = createTempDir(); Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), dir.resolve("GeoLite2-ASN.mmdb")); Files.copy(LocalDatabases.class.getResourceAsStream("/GeoLite2-City.mmdb"), dir.resolve("GeoLite2-City.mmdb")); From 673382aa5a76384f394f0e20038a0f8a8d3df916 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 7 Oct 2021 10:36:25 +0200 Subject: [PATCH 02/14] If databases are not available return a different geoip processor implementation. When DatabaseRegistry detects that databases are available then all geoip processor that failed loading will be reloaded. --- modules/ingest-geoip/build.gradle | 8 ++++ .../ingest/geoip/UpdateDatabasesIT.java | 35 ++++++++++++--- .../ingest/geoip/GeoIpDownloaderIT.java | 35 +++++++++++++++ ...gDatabasesWhilePerformingGeoLookupsIT.java | 4 +- .../ingest/geoip/DatabaseRegistry.java | 17 +++++++ .../ingest/geoip/GeoIpProcessor.java | 27 ++++++++++- .../stats/GeoIpDownloaderStatsAction.java | 45 ++++++++++++++++--- .../GeoIpDownloaderStatsTransportAction.java | 3 +- .../geoip/GeoIpProcessorFactoryTests.java | 41 +++++++++++------ ...atsActionNodeResponseSerializingTests.java | 3 +- .../IngestGeoIpClientYamlTestSuiteIT.java | 37 +++++++++++++++ .../test/ingest_geoip/20_geoip_processor.yml | 20 ++++----- .../elasticsearch/ingest/IngestService.java | 28 +++++++++++- 13 files changed, 261 insertions(+), 42 deletions(-) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 1c2cf0b4fc49d..81d7f9d53bb64 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -100,3 +100,11 @@ tasks.named("dependencyLicenses").configure { mapping from: /maxmind-db.*/, to: 'maxmind-db-reader' ignoreFile 'elastic-geoip-database-service-agreement-LICENSE.txt' } + +testClusters.configureEach { + // Needed for database downloader, uses delete-by-query to cleanup old databases from org.elasticsearch.ingest.geoip database system index + module ':modules:reindex' + // Downloader is enabled by default, but in test clusters in build disabled by default, + // but in the module downloader should be enabled by default + systemProperty 'ingest.geoip.downloader.enabled.default', 'true' +} diff --git a/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java b/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java index c6c0d3592fc67..d2322bad1bd19 100644 --- a/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java +++ b/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java @@ -22,10 +22,14 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.Map; +import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; public class UpdateDatabasesIT extends ESRestTestCase { @@ -36,7 +40,13 @@ public void test() throws Exception { simulatePipelineRequest.setJsonEntity(body); { Map response = toMap(client().performRequest(simulatePipelineRequest)); - assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Tumba")); + assertThat(ObjectPath.eval("docs.0.doc._source.tags.0", response), equalTo("_geoip_database_unavailable_GeoLite2-City.mmdb")); + } + + // Ensure no config databases have been setup: + { + Map stats = getGeoIpStatsForSingleNode(); + assertThat(stats, nullValue()); } Path configPath = PathUtils.get(System.getProperty("tests.config.dir")); @@ -46,10 +56,25 @@ public void test() throws Exception { Files.copy(UpdateDatabasesIT.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), ingestGeoipDatabaseDir.resolve("GeoLite2-City.mmdb")); - assertBusy(() -> { - Map response = toMap(client().performRequest(simulatePipelineRequest)); - assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Linköping")); - }); + // Ensure that a config database has been setup: + { + assertBusy(() -> { + Map stats = getGeoIpStatsForSingleNode(); + assertThat(stats, notNullValue()); + assertThat(stats.get("config_databases"), equalTo(List.of("GeoLite2-City.mmdb"))); + }); + } + + Map response = toMap(client().performRequest(simulatePipelineRequest)); + assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Linköping")); + } + + private static Map getGeoIpStatsForSingleNode() throws IOException { + Request request = new Request("GET", "/_ingest/geoip/stats"); + Map response = toMap(client().performRequest(request)); + Map nodes = (Map) response.get("nodes"); + assertThat(nodes.size(), either(equalTo(0)).or(equalTo(1))); + return nodes.isEmpty() ? null : (Map) nodes.values().iterator().next(); } private static Map toMap(Response response) throws IOException { diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index e2e7290fa9b68..4d24463084864 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.ingest.geoip.stats.GeoIpDownloaderStatsAction; import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; @@ -40,6 +41,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; @@ -52,6 +54,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import java.util.zip.GZIPInputStream; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -259,6 +262,8 @@ public void testGeoIpDatabasesDownload() throws Exception { @TestLogging(value = "org.elasticsearch.ingest.geoip:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/69972") public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception { assumeTrue("only test with fixture to have stable results", ENDPOINT != null); + setupDatabasesInConfigDirectory(); + Thread.sleep(1000); // setup: putPipeline(); @@ -412,6 +417,36 @@ private List getGeoIpTmpDirs() throws IOException { return geoipTmpDirs; } + private void setupDatabasesInConfigDirectory() throws Exception { + StreamSupport.stream(internalCluster().getInstances(Environment.class).spliterator(), false) + .map(Environment::configFile) + .map(path -> path.resolve("ingest-geoip")) + .distinct() + .forEach(path -> { + try { + Files.createDirectories(path); + Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-City.mmdb"), + path.resolve("GeoLite2-City.mmdb")); + Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-ASN.mmdb"), + path.resolve("GeoLite2-ASN.mmdb")); + Files.copy(GeoIpDownloaderIT.class.getResourceAsStream("/GeoLite2-Country.mmdb"), + path.resolve("GeoLite2-Country.mmdb")); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + + assertBusy(() -> { + GeoIpDownloaderStatsAction.Response response = + client().execute(GeoIpDownloaderStatsAction.INSTANCE, new GeoIpDownloaderStatsAction.Request()).actionGet(); + assertThat(response.getNodes(), not(empty())); + for (GeoIpDownloaderStatsAction.NodeResponse nodeResponse : response.getNodes()) { + assertThat(nodeResponse.getConfigDatabases(), + containsInAnyOrder("GeoLite2-Country.mmdb", "GeoLite2-City.mmdb", "GeoLite2-ASN.mmdb")); + } + }); + } + @SuppressForbidden(reason = "Maxmind API requires java.io.File") private void parseDatabase(Path tempFile) throws IOException { try (DatabaseReader databaseReader = new DatabaseReader.Builder(tempFile.toFile()).build()) { diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java index 290662738b61d..d8e184b407446 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java @@ -70,8 +70,8 @@ public void test() throws Exception { databaseRegistry.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb")); lazyLoadReaders(databaseRegistry); - final GeoIpProcessor processor1 = factory.create(null, "_tag", null, new HashMap<>(Map.of("field", "_field"))); - final GeoIpProcessor processor2 = factory.create(null, "_tag", null, + final GeoIpProcessor processor1 = (GeoIpProcessor) factory.create(null, "_tag", null, new HashMap<>(Map.of("field", "_field"))); + final GeoIpProcessor processor2 = (GeoIpProcessor) factory.create(null, "_tag", null, new HashMap<>(Map.of("field", "_field", "database_file", "GeoLite2-City-Test.mmdb"))); final AtomicBoolean completed = new AtomicBoolean(false); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java index 7562058c7983c..a765798404b05 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java @@ -53,6 +53,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -87,8 +88,10 @@ public final class DatabaseRegistry implements Closeable { private Path geoipTmpDirectory; private final LocalDatabases localDatabases; private final Consumer genericExecutor; + private IngestService ingestService; private final ConcurrentMap databases = new ConcurrentHashMap<>(); + private final AtomicBoolean initialized = new AtomicBoolean(false); DatabaseRegistry(Environment environment, Client client, GeoIpCache cache, Consumer genericExecutor) { this( @@ -150,6 +153,7 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { } LOGGER.info("initialized database registry, using geoip-databases directory [{}]", geoipTmpDirectory); ingestService.addIngestClusterStateListener(this::checkDatabases); + this.ingestService = ingestService; } public DatabaseReaderLazyLoader getDatabase(String name) { @@ -315,6 +319,15 @@ void updateDatabase(String databaseFileName, String recordedMd5, Path file) { existing.close(); } LOGGER.info("successfully reloaded changed geoip database file [{}]", file); + if (databases.size() == 3 && initialized.compareAndSet(false, true)) { + List ids = ingestService.getPipelineWithProcessorType(GeoIpProcessor.DatabaseUnavailableProcessor.class); + if (ids.isEmpty() == false) { + for (String id : ids) { + ingestService.reloadPipeline(id); + } + LOGGER.info("reloaded {} pipelines after successful initial downloading of databases", ids.size()); + } + } } catch (Exception e) { LOGGER.error((Supplier) () -> new ParameterizedMessage("failed to update database [{}]", databaseFileName), e); } @@ -384,6 +397,10 @@ public Set getAvailableDatabases() { return Set.copyOf(databases.keySet()); } + public Set getConfigDatabases() { + return localDatabases.getConfigDatabases().keySet(); + } + public Set getFilesInTemp() { try (Stream files = Files.list(geoipTmpDirectory)) { return files.map(Path::getFileName).map(Path::toString).collect(Collectors.toSet()); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 7480fcdd5b2b0..c95a441a95172 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -375,7 +375,7 @@ public Factory(DatabaseRegistry databaseRegistry, ClusterService clusterService) } @Override - public GeoIpProcessor create( + public Processor create( final Map registry, final String processorTag, final String description, final Map config) throws IOException { @@ -393,9 +393,11 @@ public GeoIpProcessor create( } DatabaseReaderLazyLoader lazyLoader = databaseRegistry.getDatabase(databaseFile); - if (lazyLoader == null) { + if (lazyLoader == null && databaseRegistry.getAvailableDatabases().isEmpty() == false) { throw newConfigurationException(TYPE, processorTag, "database_file", "database file [" + databaseFile + "] doesn't exist"); + } else if (lazyLoader == null && databaseRegistry.getAvailableDatabases().isEmpty()) { + return new DatabaseUnavailableProcessor(processorTag, description, databaseFile); } final String databaseType; try { @@ -529,4 +531,25 @@ public static Property parseProperty(String databaseType, String value) { } } } + + static class DatabaseUnavailableProcessor extends AbstractProcessor { + + private final String databaseName; + + DatabaseUnavailableProcessor(String tag, String description, String databaseName) { + super(tag, description); + this.databaseName = databaseName; + } + + @Override + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + ingestDocument.appendFieldValue("tags", "_geoip_database_unavailable_" + databaseName, true); + return ingestDocument; + } + + @Override + public String getType() { + return TYPE; + } + } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java index a9c817dbc8ad6..af44ce50e3ccb 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.ingest.geoip.stats; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.nodes.BaseNodeResponse; @@ -91,6 +92,10 @@ public Response(ClusterName clusterName, List nodes, List n.stats).filter(Objects::nonNull).findFirst().orElse(GeoIpDownloaderStats.EMPTY); + } + @Override protected List readNodesFrom(StreamInput in) throws IOException { return in.readList(NodeResponse::new); @@ -103,14 +108,13 @@ protected void writeNodesTo(StreamOutput out, List nodes) throws I @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - GeoIpDownloaderStats stats = - getNodes().stream().map(n -> n.stats).filter(Objects::nonNull).findFirst().orElse(GeoIpDownloaderStats.EMPTY); + GeoIpDownloaderStats stats = getStats(); builder.startObject(); builder.field("stats", stats); builder.startObject("nodes"); for (Map.Entry e : getNodesMap().entrySet()) { NodeResponse response = e.getValue(); - if (response.filesInTemp.isEmpty() && response.databases.isEmpty()) { + if (response.filesInTemp.isEmpty() && response.databases.isEmpty() && response.configDatabases.isEmpty()) { continue; } builder.startObject(e.getKey()); @@ -126,6 +130,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (response.filesInTemp.isEmpty() == false) { builder.array("files_in_temp", response.filesInTemp.toArray(String[]::new)); } + if (response.configDatabases.isEmpty() == false) { + builder.array("config_databases", response.configDatabases.toArray(String[]::new)); + } builder.endObject(); } builder.endObject(); @@ -152,19 +159,39 @@ public static class NodeResponse extends BaseNodeResponse { private final GeoIpDownloaderStats stats; private final Set databases; private final Set filesInTemp; + private final Set configDatabases; protected NodeResponse(StreamInput in) throws IOException { super(in); stats = in.readBoolean() ? new GeoIpDownloaderStats(in) : null; databases = in.readSet(StreamInput::readString); filesInTemp = in.readSet(StreamInput::readString); + configDatabases = in.getVersion().onOrAfter(Version.V_8_0_0) ? in.readSet(StreamInput::readString) : null; } - protected NodeResponse(DiscoveryNode node, GeoIpDownloaderStats stats, Set databases, Set filesInTemp) { + protected NodeResponse(DiscoveryNode node, GeoIpDownloaderStats stats, Set databases, Set filesInTemp, + Set configDatabases) { super(node); this.stats = stats; this.databases = databases; this.filesInTemp = filesInTemp; + this.configDatabases = configDatabases; + } + + public GeoIpDownloaderStats getStats() { + return stats; + } + + public Set getDatabases() { + return databases; + } + + public Set getFilesInTemp() { + return filesInTemp; + } + + public Set getConfigDatabases() { + return configDatabases; } @Override @@ -176,6 +203,9 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeCollection(databases, StreamOutput::writeString); out.writeCollection(filesInTemp, StreamOutput::writeString); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeCollection(configDatabases, StreamOutput::writeString); + } } @Override @@ -183,12 +213,15 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; NodeResponse that = (NodeResponse) o; - return stats.equals(that.stats) && databases.equals(that.databases) && filesInTemp.equals(that.filesInTemp); + return stats.equals(that.stats) && + databases.equals(that.databases) && + filesInTemp.equals(that.filesInTemp) && + Objects.equals(configDatabases, that.configDatabases); } @Override public int hashCode() { - return Objects.hash(stats, databases, filesInTemp); + return Objects.hash(stats, databases, filesInTemp, configDatabases); } } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java index fe73f98b68c93..d39f99aa71512 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsTransportAction.java @@ -66,6 +66,7 @@ protected NodeResponse newNodeResponse(StreamInput in, DiscoveryNode node) throw protected NodeResponse nodeOperation(NodeRequest request, Task task) { GeoIpDownloader geoIpTask = geoIpDownloaderTaskExecutor.getCurrentTask(); GeoIpDownloaderStats stats = geoIpTask == null || geoIpTask.getStatus() == null ? null : geoIpTask.getStatus(); - return new NodeResponse(transportService.getLocalNode(), stats, registry.getAvailableDatabases(), registry.getFilesInTemp()); + return new NodeResponse(transportService.getLocalNode(), stats, registry.getAvailableDatabases(), registry.getFilesInTemp(), + registry.getConfigDatabases()); } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index 36d036cfaacfb..bc14e58886561 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.RandomDocumentPicks; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.test.ESTestCase; @@ -46,6 +47,7 @@ import java.util.Set; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -87,7 +89,7 @@ public void testBuildDefaults() throws Exception { config.put("field", "_field"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = factory.create(null, processorTag, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("geoip")); @@ -104,7 +106,7 @@ public void testSetIgnoreMissing() throws Exception { config.put("ignore_missing", true); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = factory.create(null, processorTag, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("geoip")); @@ -121,7 +123,7 @@ public void testCountryBuildDefaults() throws Exception { config.put("database_file", "GeoLite2-Country.mmdb"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = factory.create(null, processorTag, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); @@ -139,7 +141,7 @@ public void testAsnBuildDefaults() throws Exception { config.put("database_file", "GeoLite2-ASN.mmdb"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = factory.create(null, processorTag, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); @@ -154,7 +156,7 @@ public void testBuildTargetField() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("target_field", "_field"); - GeoIpProcessor processor = factory.create(null, null, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("_field")); assertFalse(processor.isIgnoreMissing()); @@ -165,7 +167,7 @@ public void testBuildDbFile() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-Country.mmdb"); - GeoIpProcessor processor = factory.create(null, null, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getTargetField(), equalTo("geoip")); assertThat(processor.getDatabaseType(), equalTo("GeoLite2-Country")); @@ -202,6 +204,9 @@ public void testBuildWithAsnDbAndCityFields() throws Exception { } public void testBuildNonExistingDbFile() throws Exception { + Files.copy(GeoIpProcessorFactoryTests.class.getResourceAsStream("/GeoLite2-City-Test.mmdb"), + geoipTmpDir.resolve("GeoLite2-City.mmdb")); + databaseRegistry.updateDatabase("GeoLite2-City.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City.mmdb")); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry, clusterService); Map config = new HashMap<>(); @@ -211,6 +216,16 @@ public void testBuildNonExistingDbFile() throws Exception { assertThat(e.getMessage(), equalTo("[database_file] database file [does-not-exist.mmdb] doesn't exist")); } + public void testBuildNoDatabasesDownloaded() throws Exception { + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry, clusterService); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("database_file", "does-not-exist-yet.mmdb"); + Processor processor = factory.create(null, null, null, config); + assertThat(processor, instanceOf(GeoIpProcessor.DatabaseUnavailableProcessor.class)); + } + public void testBuildFields() throws Exception { GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry, clusterService); @@ -229,7 +244,7 @@ public void testBuildFields() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", fieldNames); - GeoIpProcessor processor = factory.create(null, null, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(properties)); assertFalse(processor.isIgnoreMissing()); @@ -276,7 +291,7 @@ public void testLazyLoading() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-City.mmdb"); - final GeoIpProcessor city = factory.create(null, "_tag", null, config); + final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here assertNull(databaseRegistry.getDatabase("GeoLite2-City.mmdb").databaseReader.get()); @@ -287,7 +302,7 @@ public void testLazyLoading() throws Exception { config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-Country.mmdb"); - final GeoIpProcessor country = factory.create(null, "_tag", null, config); + final GeoIpProcessor country = (GeoIpProcessor) factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here assertNull(databaseRegistry.getDatabase("GeoLite2-Country.mmdb").databaseReader.get()); @@ -298,7 +313,7 @@ public void testLazyLoading() throws Exception { config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoLite2-ASN.mmdb"); - final GeoIpProcessor asn = factory.create(null, "_tag", null, config); + final GeoIpProcessor asn = (GeoIpProcessor) factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here assertNull(databaseRegistry.getDatabase("GeoLite2-ASN.mmdb").databaseReader.get()); @@ -338,7 +353,7 @@ public void testLoadingCustomDatabase() throws IOException { Map config = new HashMap<>(); config.put("field", "_field"); config.put("database_file", "GeoIP2-City.mmdb"); - final GeoIpProcessor city = factory.create(null, "_tag", null, config); + final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config); // these are lazy loaded until first use so we expect null here assertNull(databaseRegistry.getDatabase("GeoIP2-City.mmdb").databaseReader.get()); @@ -373,7 +388,7 @@ public void testDefaultDatabaseWithTaskPresent() throws Exception { config.put("field", "_field"); String processorTag = randomAlphaOfLength(10); - GeoIpProcessor processor = factory.create(null, processorTag, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config); processor.execute(RandomDocumentPicks.randomIngestDocument(random(), Map.of("_field", "89.160.20.128"))); } @@ -382,7 +397,7 @@ public void testUpdateDatabaseWhileIngesting() throws Exception { GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry, clusterService); Map config = new HashMap<>(); config.put("field", "source_field"); - GeoIpProcessor processor = factory.create(null, null, null, config); + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); Map document = new HashMap<>(); document.put("source_field", "89.160.20.128"); { diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java index 9a5046e503d58..cb914eee8f893 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpDownloaderStatsActionNodeResponseSerializingTests.java @@ -32,7 +32,8 @@ static GeoIpDownloaderStatsAction.NodeResponse createRandomInstance() { DiscoveryNode node = new DiscoveryNode("id", buildNewFakeTransportAddress(), Version.CURRENT); Set databases = Set.copyOf(randomList(10, () -> randomAlphaOfLengthBetween(5, 10))); Set files = Set.copyOf(randomList(10, () -> randomAlphaOfLengthBetween(5, 10))); + Set configDatabases = Set.copyOf(randomList(10, () -> randomAlphaOfLengthBetween(5, 10))); return new GeoIpDownloaderStatsAction.NodeResponse(node, GeoIpDownloaderStatsSerializingTests.createRandomInstance(), databases, - files); + files, configDatabases); } } diff --git a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java index 4f97b67c7666c..1981d1948d592 100644 --- a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java +++ b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java @@ -11,8 +11,22 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; public class IngestGeoIpClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { @@ -24,4 +38,27 @@ public IngestGeoIpClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate te public static Iterable parameters() throws Exception { return ESClientYamlSuiteTestCase.createParameters(); } + + @Before + public void waitForDatabases() throws Exception { + assertBusy(() -> { + Request request = new Request("GET", "/_ingest/geoip/stats"); + Map response = toMap(client().performRequest(request)); + + Map downloadStats = (Map) response.get("stats"); + assertThat(downloadStats.get("databases_count"), equalTo(3)); + + Map nodes = (Map) response.get("nodes"); + assertThat(nodes.size(), equalTo(1)); + Map node = (Map) nodes.values().iterator().next(); + List databases = ((List) node.get("databases")).stream() + .map(o -> (String) ((Map) o).get("name")) + .collect(Collectors.toList()); + assertThat(databases, containsInAnyOrder("GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "GeoLite2-ASN.mmdb")); + }); + } + + private static Map toMap(Response response) throws IOException { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false); + } } diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml index 3b5ee3bd55381..d1f48d17e6764 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml @@ -31,8 +31,8 @@ - length: { _source.geoip: 7 } - match: { _source.geoip.city_name: "Minneapolis" } - match: { _source.geoip.country_iso_code: "US" } - - match: { _source.geoip.location.lon: -93.2548 } - - match: { _source.geoip.location.lat: 44.9399 } + - match: { _source.geoip.location.lon: -93.2566 } + - match: { _source.geoip.location.lat: 45.0546 } - match: { _source.geoip.region_iso_code: "US-MN" } - match: { _source.geoip.country_name: "United States" } - match: { _source.geoip.region_name: "Minnesota" } @@ -73,8 +73,8 @@ - length: { _source.geoip.0: 7 } - match: { _source.geoip.0.city_name: "Minneapolis" } - match: { _source.geoip.0.country_iso_code: "US" } - - match: { _source.geoip.0.location.lon: -93.2548 } - - match: { _source.geoip.0.location.lat: 44.9399 } + - match: { _source.geoip.0.location.lon: -93.2566 } + - match: { _source.geoip.0.location.lat: 45.0546 } - match: { _source.geoip.0.region_iso_code: "US-MN" } - match: { _source.geoip.0.country_name: "United States" } - match: { _source.geoip.0.region_name: "Minnesota" } @@ -114,8 +114,8 @@ - length: { _source.geoip: 7 } - match: { _source.geoip.city_name: "Minneapolis" } - match: { _source.geoip.country_iso_code: "US" } - - match: { _source.geoip.location.lon: -93.2548 } - - match: { _source.geoip.location.lat: 44.9399 } + - match: { _source.geoip.location.lon: -93.2566 } + - match: { _source.geoip.location.lat: 45.0546 } - match: { _source.geoip.region_iso_code: "US-MN" } - match: { _source.geoip.country_name: "United States" } - match: { _source.geoip.region_name: "Minnesota" } @@ -160,8 +160,8 @@ - match: { _source.geoip.city_name: "Minneapolis" } - match: { _source.geoip.country_iso_code: "US" } - match: { _source.geoip.ip: "128.101.101.101" } - - match: { _source.geoip.location.lon: -93.2548 } - - match: { _source.geoip.location.lat: 44.9399 } + - match: { _source.geoip.location.lon: -93.2566 } + - match: { _source.geoip.location.lat: 45.0546 } - match: { _source.geoip.timezone: "America/Chicago" } - match: { _source.geoip.country_name: "United States" } - match: { _source.geoip.region_iso_code: "US-MN" } @@ -266,8 +266,8 @@ - length: { _source.geoip: 7 } - match: { _source.geoip.city_name: "Minneapolis" } - match: { _source.geoip.country_iso_code: "US" } - - match: { _source.geoip.location.lon: -93.2548 } - - match: { _source.geoip.location.lat: 44.9399 } + - match: { _source.geoip.location.lon: -93.2566 } + - match: { _source.geoip.location.lat: 45.0546 } - match: { _source.geoip.region_iso_code: "US-MN" } - match: { _source.geoip.country_name: "United States" } - match: { _source.geoip.region_name: "Minnesota" } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 98ea8a351e607..2b169d15075ee 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -57,6 +57,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -740,7 +741,7 @@ public void applyClusterState(final ClusterChangedEvent event) { } } - void innerUpdatePipelines(IngestMetadata newIngestMetadata) { + synchronized void innerUpdatePipelines(IngestMetadata newIngestMetadata) { Map existingPipelines = this.pipelines; // Lazy initialize these variables in order to favour the most like scenario that there are no pipeline changes: @@ -839,7 +840,7 @@ void innerUpdatePipelines(IngestMetadata newIngestMetadata) { * @param clazz the Processor class to look for * @return True if the pipeline contains an instance of the Processor class passed in */ - public

List

getProcessorsInPipeline(String pipelineId, Class

clazz) { + public

List

getProcessorsInPipeline(String pipelineId, Class

clazz) { Pipeline pipeline = getPipeline(pipelineId); if (pipeline == null) { throw new IllegalArgumentException("pipeline with id [" + pipelineId + "] does not exist"); @@ -868,6 +869,29 @@ public

List

getProcessorsInPipeline(String pipelineId, C return processors; } + public

List getPipelineWithProcessorType(Class

clazz) { + List matchedPipelines = new LinkedList<>(); + for (PipelineHolder holder : pipelines.values()) { + String pipelineId = holder.pipeline.getId(); + List

processors = getProcessorsInPipeline(pipelineId, clazz); + if (processors.isEmpty() == false) { + matchedPipelines.add(pipelineId); + } + } + return matchedPipelines; + } + + public void reloadPipeline(String id) throws Exception { + PipelineHolder holder = pipelines.get(id); + + Pipeline updatedPipeline = + Pipeline.create(id, holder.configuration.getConfigAsMap(), processorFactories, scriptService); + synchronized (this) { + Map existingPipelines = this.pipelines; + existingPipelines.put(id, new PipelineHolder(holder.configuration, updatedPipeline)); + } + } + private static Pipeline substitutePipeline(String id, ElasticsearchParseException e) { String tag = e.getHeaderKeys().contains("processor_tag") ? e.getHeader("processor_tag").get(0) : null; String type = e.getHeaderKeys().contains("processor_type") ? e.getHeader("processor_type").get(0) : "unknown"; From b44dcb779ed20d230d5889020fcb6455ceb5d21b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 7 Oct 2021 17:05:09 +0200 Subject: [PATCH 03/14] transform 7.x geoip yaml tests in order to work with downloaded geoip databases --- modules/ingest-geoip/build.gradle | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 81d7f9d53bb64..537179ce0f279 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -108,3 +108,11 @@ testClusters.configureEach { // but in the module downloader should be enabled by default systemProperty 'ingest.geoip.downloader.enabled.default', 'true' } + +// 7.x yaml tests need to use different lat/lon that match databases from geoip-fixture: +tasks.named("yamlRestTestV7CompatTransform").configure { task -> + task.replaceValueInMatch("_source.geoip.location.lon", -93.2566) + task.replaceValueInMatch("_source.geoip.0.location.lon", -93.2566) + task.replaceValueInMatch("_source.geoip.location.lat", 45.0546) + task.replaceValueInMatch("_source.geoip.0.location.lat", 45.0546) +} From 9310ec07d4137b0764e5590d3b1817a936f773e0 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 8 Oct 2021 00:04:30 +0200 Subject: [PATCH 04/14] use geoip fixture for yaml tests and ignore a few 7.x rest backwards compatability tests because these assume default database and not fixture. --- modules/ingest-geoip/build.gradle | 13 +- .../ingest/geoip/UpdateDatabasesIT.java | 16 +-- .../ingest/geoip/GeoIpDownloaderIT.java | 1 - .../IngestGeoIpClientYamlTestSuiteIT.java | 10 +- .../test/ingest_geoip/20_geoip_processor.yml | 126 +++++++++--------- 5 files changed, 74 insertions(+), 92 deletions(-) diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 537179ce0f279..940fec7cff471 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -13,7 +13,7 @@ apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' esplugin { - description 'Ingest processor that uses lookup geo data based on IP adresses using the MaxMind geo database' + description 'Ingest processor that uses lookup geo data based on IP addresses using the MaxMind geo database' classname 'org.elasticsearch.ingest.geoip.IngestGeoIpPlugin' } @@ -105,14 +105,13 @@ testClusters.configureEach { // Needed for database downloader, uses delete-by-query to cleanup old databases from org.elasticsearch.ingest.geoip database system index module ':modules:reindex' // Downloader is enabled by default, but in test clusters in build disabled by default, - // but in the module downloader should be enabled by default + // but in this module, the downloader should be enabled by default systemProperty 'ingest.geoip.downloader.enabled.default', 'true' + if (useFixture) { + setting 'ingest.geoip.downloader.endpoint', { "${-> fixtureAddress()}" } + } } -// 7.x yaml tests need to use different lat/lon that match databases from geoip-fixture: tasks.named("yamlRestTestV7CompatTransform").configure { task -> - task.replaceValueInMatch("_source.geoip.location.lon", -93.2566) - task.replaceValueInMatch("_source.geoip.0.location.lon", -93.2566) - task.replaceValueInMatch("_source.geoip.location.lat", 45.0546) - task.replaceValueInMatch("_source.geoip.0.location.lat", 45.0546) + task.skipTestsByFilePattern("**/ingest_geoip/20_geoip_processor.yml", "from 8.0 yaml rest tests use geoip test fixture and default geoip are no longer packaged. In 7.x yaml tests used default databases which makes tests results very different, so skipping these tests") } diff --git a/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java b/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java index d2322bad1bd19..66503fab77403 100644 --- a/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java +++ b/modules/ingest-geoip/qa/file-based-update/src/test/java/org/elasticsearch/ingest/geoip/UpdateDatabasesIT.java @@ -7,16 +7,12 @@ */ package org.elasticsearch.ingest.geoip; -import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.core.PathUtils; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.ObjectPath; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.core.PathUtils; import org.elasticsearch.test.rest.ESRestTestCase; import java.io.IOException; @@ -39,7 +35,7 @@ public void test() throws Exception { Request simulatePipelineRequest = new Request("POST", "/_ingest/pipeline/_simulate"); simulatePipelineRequest.setJsonEntity(body); { - Map response = toMap(client().performRequest(simulatePipelineRequest)); + Map response = entityAsMap(client().performRequest(simulatePipelineRequest)); assertThat(ObjectPath.eval("docs.0.doc._source.tags.0", response), equalTo("_geoip_database_unavailable_GeoLite2-City.mmdb")); } @@ -65,22 +61,18 @@ public void test() throws Exception { }); } - Map response = toMap(client().performRequest(simulatePipelineRequest)); + Map response = entityAsMap(client().performRequest(simulatePipelineRequest)); assertThat(ObjectPath.eval("docs.0.doc._source.geoip.city_name", response), equalTo("Linköping")); } private static Map getGeoIpStatsForSingleNode() throws IOException { Request request = new Request("GET", "/_ingest/geoip/stats"); - Map response = toMap(client().performRequest(request)); + Map response = entityAsMap(client().performRequest(request)); Map nodes = (Map) response.get("nodes"); assertThat(nodes.size(), either(equalTo(0)).or(equalTo(1))); return nodes.isEmpty() ? null : (Map) nodes.values().iterator().next(); } - private static Map toMap(Response response) throws IOException { - return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false); - } - @Override protected Settings restClientSettings() { String token = basicAuthHeaderValue("admin", new SecureString("admin-password".toCharArray())); diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index 4d24463084864..4648802726da6 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -263,7 +263,6 @@ public void testGeoIpDatabasesDownload() throws Exception { public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception { assumeTrue("only test with fixture to have stable results", ENDPOINT != null); setupDatabasesInConfigDirectory(); - Thread.sleep(1000); // setup: putPipeline(); diff --git a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java index 1981d1948d592..5b40f4a6ada43 100644 --- a/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java +++ b/modules/ingest-geoip/src/yamlRestTest/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java @@ -11,16 +11,11 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.junit.Before; -import java.io.IOException; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -43,7 +38,7 @@ public static Iterable parameters() throws Exception { public void waitForDatabases() throws Exception { assertBusy(() -> { Request request = new Request("GET", "/_ingest/geoip/stats"); - Map response = toMap(client().performRequest(request)); + Map response = entityAsMap(client().performRequest(request)); Map downloadStats = (Map) response.get("stats"); assertThat(downloadStats.get("databases_count"), equalTo(3)); @@ -58,7 +53,4 @@ public void waitForDatabases() throws Exception { }); } - private static Map toMap(Response response) throws IOException { - return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false); - } } diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml index d1f48d17e6764..d0da405cdeea9 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yml @@ -21,22 +21,22 @@ index: test id: 1 pipeline: "my_pipeline" - body: {field1: "128.101.101.101"} + body: {field1: "89.160.20.128"} - do: get: index: test id: 1 - - match: { _source.field1: "128.101.101.101" } + - match: { _source.field1: "89.160.20.128" } - length: { _source.geoip: 7 } - - match: { _source.geoip.city_name: "Minneapolis" } - - match: { _source.geoip.country_iso_code: "US" } - - match: { _source.geoip.location.lon: -93.2566 } - - match: { _source.geoip.location.lat: 45.0546 } - - match: { _source.geoip.region_iso_code: "US-MN" } - - match: { _source.geoip.country_name: "United States" } - - match: { _source.geoip.region_name: "Minnesota" } - - match: { _source.geoip.continent_name: "North America" } + - match: { _source.geoip.city_name: "Linköping" } + - match: { _source.geoip.country_iso_code: "SE" } + - match: { _source.geoip.location.lon: 15.6167 } + - match: { _source.geoip.location.lat: 58.4167 } + - match: { _source.geoip.region_iso_code: "SE-E" } + - match: { _source.geoip.country_name: "Sweden" } + - match: { _source.geoip.region_name: "Östergötland County" } + - match: { _source.geoip.continent_name: "Europe" } --- "Test geoip processor with list": @@ -62,23 +62,23 @@ index: test id: 1 pipeline: "my_pipeline" - body: {field1: ["128.101.101.101", "127.0.0.1"]} + body: {field1: ["89.160.20.128", "127.0.0.1"]} - do: get: index: test id: 1 - - match: { _source.field1: ["128.101.101.101", "127.0.0.1"] } + - match: { _source.field1: ["89.160.20.128", "127.0.0.1"] } - length: { _source.geoip: 2 } - length: { _source.geoip.0: 7 } - - match: { _source.geoip.0.city_name: "Minneapolis" } - - match: { _source.geoip.0.country_iso_code: "US" } - - match: { _source.geoip.0.location.lon: -93.2566 } - - match: { _source.geoip.0.location.lat: 45.0546 } - - match: { _source.geoip.0.region_iso_code: "US-MN" } - - match: { _source.geoip.0.country_name: "United States" } - - match: { _source.geoip.0.region_name: "Minnesota" } - - match: { _source.geoip.0.continent_name: "North America" } + - match: { _source.geoip.0.city_name: "Linköping" } + - match: { _source.geoip.0.country_iso_code: "SE" } + - match: { _source.geoip.0.location.lon: 15.6167 } + - match: { _source.geoip.0.location.lat: 58.4167 } + - match: { _source.geoip.0.region_iso_code: "SE-E" } + - match: { _source.geoip.0.country_name: "Sweden" } + - match: { _source.geoip.0.region_name: "Östergötland County" } + - match: { _source.geoip.0.continent_name: "Europe" } - match: { _source.geoip.1: null } --- @@ -104,22 +104,22 @@ index: test id: 1 pipeline: "my_pipeline" - body: {field1: ["127.0.0.1", "128.101.101.101", "128.101.101.101"]} + body: {field1: ["127.0.0.1", "89.160.20.128", "89.160.20.128"]} - do: get: index: test id: 1 - - match: { _source.field1: ["127.0.0.1", "128.101.101.101", "128.101.101.101"] } + - match: { _source.field1: ["127.0.0.1", "89.160.20.128", "89.160.20.128"] } - length: { _source.geoip: 7 } - - match: { _source.geoip.city_name: "Minneapolis" } - - match: { _source.geoip.country_iso_code: "US" } - - match: { _source.geoip.location.lon: -93.2566 } - - match: { _source.geoip.location.lat: 45.0546 } - - match: { _source.geoip.region_iso_code: "US-MN" } - - match: { _source.geoip.country_name: "United States" } - - match: { _source.geoip.region_name: "Minnesota" } - - match: { _source.geoip.continent_name: "North America" } + - match: { _source.geoip.city_name: "Linköping" } + - match: { _source.geoip.country_iso_code: "SE" } + - match: { _source.geoip.location.lon: 15.6167 } + - match: { _source.geoip.location.lat: 58.4167 } + - match: { _source.geoip.region_iso_code: "SE-E" } + - match: { _source.geoip.country_name: "Sweden" } + - match: { _source.geoip.region_name: "Östergötland County" } + - match: { _source.geoip.continent_name: "Europe" } --- "Test geoip processor with fields": @@ -149,24 +149,24 @@ index: test id: 1 pipeline: "my_pipeline" - body: {field1: "128.101.101.101"} + body: {field1: "89.160.20.128"} - do: get: index: test id: 1 - - match: { _source.field1: "128.101.101.101" } + - match: { _source.field1: "89.160.20.128" } - length: { _source.geoip: 9 } - - match: { _source.geoip.city_name: "Minneapolis" } - - match: { _source.geoip.country_iso_code: "US" } - - match: { _source.geoip.ip: "128.101.101.101" } - - match: { _source.geoip.location.lon: -93.2566 } - - match: { _source.geoip.location.lat: 45.0546 } - - match: { _source.geoip.timezone: "America/Chicago" } - - match: { _source.geoip.country_name: "United States" } - - match: { _source.geoip.region_iso_code: "US-MN" } - - match: { _source.geoip.region_name: "Minnesota" } - - match: { _source.geoip.continent_name: "North America" } + - match: { _source.geoip.city_name: "Linköping" } + - match: { _source.geoip.country_iso_code: "SE" } + - match: { _source.geoip.ip: "89.160.20.128" } + - match: { _source.geoip.location.lon: 15.6167 } + - match: { _source.geoip.location.lat: 58.4167 } + - match: { _source.geoip.timezone: "Europe/Stockholm" } + - match: { _source.geoip.country_name: "Sweden" } + - match: { _source.geoip.region_iso_code: "SE-E" } + - match: { _source.geoip.region_name: "Östergötland County" } + - match: { _source.geoip.continent_name: "Europe" } --- "Test geoip processor with different database file - GeoLite2-Country": @@ -192,17 +192,17 @@ index: test id: 1 pipeline: "my_pipeline" - body: {field1: "128.101.101.101"} + body: {field1: "89.160.20.128"} - do: get: index: test id: 1 - - match: { _source.field1: "128.101.101.101" } + - match: { _source.field1: "89.160.20.128" } - length: { _source.geoip: 3 } - - match: { _source.geoip.country_iso_code: "US" } - - match: { _source.geoip.country_name: "United States" } - - match: { _source.geoip.continent_name: "North America" } + - match: { _source.geoip.country_iso_code: "SE" } + - match: { _source.geoip.country_name: "Sweden" } + - match: { _source.geoip.continent_name: "Europe" } --- "Test geoip processor with geopoint mapping (both missing and including location)": @@ -256,22 +256,22 @@ index: test id: 2 pipeline: "my_pipeline" - body: { field1: "128.101.101.101" } + body: { field1: "89.160.20.128" } - do: get: index: test id: 2 - - match: { _source.field1: "128.101.101.101" } + - match: { _source.field1: "89.160.20.128" } - length: { _source.geoip: 7 } - - match: { _source.geoip.city_name: "Minneapolis" } - - match: { _source.geoip.country_iso_code: "US" } - - match: { _source.geoip.location.lon: -93.2566 } - - match: { _source.geoip.location.lat: 45.0546 } - - match: { _source.geoip.region_iso_code: "US-MN" } - - match: { _source.geoip.country_name: "United States" } - - match: { _source.geoip.region_name: "Minnesota" } - - match: { _source.geoip.continent_name: "North America" } + - match: { _source.geoip.city_name: "Linköping" } + - match: { _source.geoip.country_iso_code: "SE" } + - match: { _source.geoip.location.lon: 15.6167 } + - match: { _source.geoip.location.lat: 58.4167 } + - match: { _source.geoip.region_iso_code: "SE-E" } + - match: { _source.geoip.country_name: "Sweden" } + - match: { _source.geoip.region_name: "Östergötland County" } + - match: { _source.geoip.continent_name: "Europe" } --- "Test geoip processor with different database file - GeoLite2-ASN": @@ -297,15 +297,15 @@ index: test id: 1 pipeline: "my_pipeline" - body: {field1: "82.171.64.0"} + body: {field1: "89.160.20.128"} - do: get: index: test id: 1 - - match: { _source.field1: "82.171.64.0" } + - match: { _source.field1: "89.160.20.128" } - length: { _source.geoip: 4 } - - match: { _source.geoip.ip: "82.171.64.0" } - - match: { _source.geoip.asn: 1136 } - - match: { _source.geoip.organization_name: "KPN B.V." } - - match: { _source.geoip.network: "82.168.0.0/14" } + - match: { _source.geoip.ip: "89.160.20.128" } + - match: { _source.geoip.asn: 29518 } + - match: { _source.geoip.organization_name: "Bredband2 AB" } + - match: { _source.geoip.network: "89.160.0.0/17" } From a5cc701d71eb59885377a7425d76438bdda0e4c0 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 8 Oct 2021 15:15:31 +0200 Subject: [PATCH 05/14] adjust ingest qa module to use test database from config directory --- .../build.gradle | 5 +++++ .../src/test/resources/GeoLite2-City.mmdb | Bin 0 -> 20809 bytes .../test/ingest/20_combine_processors.yml | 16 ++++++++-------- 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 qa/smoke-test-ingest-with-all-dependencies/src/test/resources/GeoLite2-City.mmdb diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle index 1654b1377866d..3df6a245ab5dc 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle +++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle @@ -21,6 +21,7 @@ dependencies { testClusters.configureEach { setting 'xpack.security.enabled', 'false' + extraConfigFile 'ingest-geoip/GeoLite2-City.mmdb', file("${project.projectDir}/src/test/resources/GeoLite2-City.mmdb") } tasks.named("testingConventions").configure { @@ -30,3 +31,7 @@ tasks.named("testingConventions").configure { } } } + +tasks.named("forbiddenPatterns").configure { + exclude '**/*.mmdb' +} diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/GeoLite2-City.mmdb b/qa/smoke-test-ingest-with-all-dependencies/src/test/resources/GeoLite2-City.mmdb new file mode 100644 index 0000000000000000000000000000000000000000..0809201619b5915fa6189200e4c53e6088f437da GIT binary patch literal 20809 zcmZ{q349bq_Qz|wdjL^XKu|G-C|o2F8tyn;17UZeiTYxQ%f;;||81jJp_jGsZF!8A*&}MhYX9k;ZT_ z(iv_>1|yS^#mHvlFvc-*8TX+1e=kHHr}7yEj6%kE#stPh#=VS5jLD2CjH!(K7)6Zx z86F|TyFyF@YK16fVF|;8@IfJ_bE*`nGKyju$zB%DAV`6jNqM52QNi#r{6bVlOU`0p zK!{*eSS3V=@A&{@He(K>nm`Yl%j&-q0*-=&gnuB|8-@5|R9M48SXT!FUlpPrK4m*)M6o)L{m#yZy95`5Gz>y(P)vCoVSXxT8K52*9|w4 zmOloQ4hr!&r`ARF)(i0j3!h|cU~FV;Vl)uQ1)GJ~!osal;Wi<*vkH1(noFL9ASF3c$4)ajONZfJdB?AHsddhcL*}$J>2ZSEc~kw?|15nzX|bo{B7y>1N^_ql58Jf-Hsq65?M%e8zeIW_%va`$C8>S@;#> zKa8)VMZV#?d@ICvocBHBhiH)>xyVn19%F>~8DkcD<1YZ5{40g_d*s@ue6BUkQ zyXM2!~{E<<^sAbeK>KP9)9%k_TXM2QG^BD^m3kl+WVOz{5EnzHW zEMqKZtYAFKSjkw$Sj||&;QnWOj8l&@)-l#Io?!6&XXE+LwvmOK7!8cgj4h05~07)KeH`_N*x}x{>*rl@fzcG#v6<`ahJ*5(h*^6 zX5m|mw;6w7yn~Xt!uBqw-b1%PBW!==)cXv~ebT_cGd^H^$oPoy4+6r8N`B1v1h>9K z*jjMUZ-nhE(7`6PGTMZ#olBl0Tnt;}gZ_z<{}#4SIrT5bXM~=r7k`c-UkTe6ocCp1 z-hWv5HKEsCocb2F*oEyo==~&Y-?Q)s#*YN+>HkGZP1t@$$zMgU2=EI5mT3ZcHY+9b zq)=xVv3JqBpkxoBbp^T$ts56PFRsY>EW992cp(&e3hg4!yO^LVt@UDI?>L1%P`E*8 zeSxclb}2AGXqN%~h1QQ1E|0s*6)e1xpxmPkgu)=9IXEvNPGK+$hY*yPw4qQKDYRk0 z)k1S}k>PPgu3_PbIN`NWxL#2?<05i3Od2<C(DKr0YhVN^Jtg%cPPg?2CBY*J?tsmpy#o7ySV?h{%OmsEKk zPN_N+bBcS8#`7N;Fq7ej^iBxvLEuTD{SjCy zv>MscLn@_7$4<`Mc{&#C!PSR}LsoLb0v%4dsN$n)Q&8mE>)VU5t1 z11p8Lf`yL~E?vVZp8vGfl(K3o!)=|=9^)d96E0JM!tR<}w0EfFfD%@CkMUQ6Rh7R%;X|SE{HJ|Dd6r*3VugPY25>NFA4B15p?v~$2(1NZ z6WUoW*-EfXYG>g&f^|>Kf7+Kq`xN-M(Ei0mc>c3Y`kaMd5UiVh#lrs(29Dv>H@M4x zh4wA*gV4TX;r9ef??)E?MChaX(a&_Ssr|xv9B!(?bsGtF?kmb_-GD-}(Cxqwp?3lL z2)!$Cq0qYl=L`Kj*6U6f$P=`F0SkK&RR7a2frNpP@|1r2!$y^9}nCs^a)&KB4H3ipTxq+1j~_Ap>V&@@8i58 zg4NO<7EU8rl|~QOgF-ifa-mNLyh1PKl4WrvG5=w})n^jaT%}h)p;G8R&hy79%wl1H zFj&QB6%>9a^bjyd=nrs_*#xT}@%*RHC86a(Isg4p=zm~^2MJcCxo_&Vad*M|XFMYG zhXC}khoSzm(B}c$g#HMyM(Fc_WkO#7EEYP?fBGW2i`9LX@bi`u1|R3tawx16`U*}x zN*IEBO82Z{tS0nPmaT=t2B9OM_4PvM`A=U*MXWmT{HH%jLN%G{8==*hDV6V`30#6Bj7c1;0SeQ&j}qq(@zQg1Pf0R`u)nO(~M^c)|mA?6kZg1nDbsBT&C{% z5)1!Cu-fPqD4Y}epMm#<{wna6&|d@I6#DC|@CL!^2oV-G6Ri5Z4TX1w{uj=BhhTM* z_gMH>g5~SKLE&Se{~h>9=pS&A4+)lK|6t)6!q7`O^$8SOh2FxcvjnT|Z7ggjSWoYO z!hePSPv9G&e+qmh^nU@L3;i=z_&33N>=!KjGVU(_Vd2+tCBMa8eh~V1ocBFpsA`oT zS@;vdvgK!VCqwAJ05+j>Ga7=53|mVuG=@%4-Zkt{=q8LVoY$40y0&p13%e6k=o=S6 zAzc_ffZ@Wp5V%|znE#Bv!nhddEsUP5*Nb31=n@w8;hQO2*z?9^oX7K@s*%wjHyb33 zD}aH*xRQ$uAPmRT5y^&wkw8#0i7^-oLxnMf^R6POQPCL2LMK5*opCi3ZWYEgz-VEN z07eO8B$vFF(5Hq|*D^gkVL_bQYEp)Np8cp)f}nGk{sbmXggNoZ9J^PdqCMvxV% z2x<&5c>XhHlh6tj%zs9WFy;b(5XSFV@Am}jo;?2L3T= z;4xvW;Ua4ZmX~<`GuDyNin=Gb^NzFd1i@;*Q&4zO7^i{fh4CyGd5&PkZkUBH5Ui)a1cg_G@h8rEnP7SF&n$eE zV2uo~L*WZyya9YDj5mRIg%JVX7Dh8GaCf>|)%Gtee1~A&`aLN8O&EXWy!Q!~oquQH z2L!d%Gd_YshcNyDd?Ji9T;yYd<$@OEwF%=a=d}{7=e4u&T%5u`q41e7KIOcB5w7{1 zQ~zdg%IaKSLI|C&@c##4VE!|{6~@;^%te(haY*p;B#$bKGD7YKWI&O4u=#sYf}7G6lO6fQ;zuaSBJ z*9m(sAVJuB19+Wx32>>f_W|&dr!PTSZNH3#{Rpb=_Wno>5cVrLkLN!XZuWsJbP%kk z4?=3Vunz`?3i}W)auq@K2m3Gp7*Sa>(Vs#qdY>B61_qzZd7 z7fB&ldTA_l5mdC=-AH8#dj{ua64Zj#p3TA>g7u(Wr2N8u4{*P*=K=Q$dp>}p&H@1S zD;V9;`hrx%UPZ8ikLN%8Y!X^_RwMO$ zVV}zizav=w`~wRgBv=YHNa1x}Er3^jbzG#LVCku<%p>7QUM1M)Beh7_7qG%YR^axt zWB!xvu!L|ecNqwToxgvvFPGWEzJl>6gTH^V2oFd=S=qpO5DYMUzOKq`l?z|iv1;KQfi^2Yq%&d!qN9! zQi++C8VdL;Jy75)Gsz4(TuIXdj(d`3c*eEV?`Ww%(Ne#srM|wUeno0!)ySkg0tl)bD`2#1FGF^ZeDGP=z@X z0vMRpQn$6G?&+5LT`hG-9U;d#ST-f{Xylp5@yLdbz*YQ|0yh&W`gG7l#6#;_9M+r^FiyHk727Bd01WTJ_m%j4X_fw9m8bk;$C%`C6R*z;n5Eue6)Ic;Cy-yx!Y|i$ z-(KJi7I}R|ru4t*@fC%9Fvix^u`?+xkNuR97V!9HTCPcxsbbkaF1qr1;EseZ;e^7L z`qeGPB4R2jwCZMGg#&gm@s3i7Fi1`5HXR1uw_x?s3Wo>@(fHn);u4E)i{=9 zo!fT$?1{at4=?JdZ)kNmTO4DO67!{9fos?;sfj_a>BId~aDm7T4%T6gOqg(nyQOY( zOWhItb%avGwX#GYlpa|gIjkyv_SiZEsbifqxc8Np!{12-sxKs!g{nOA&V%!D+q9%m zRk*>2dm}l+lh;zeM)rle({ObA(naKADzn9pM`T1+G}m*SA;!)gS#tKs(S)&Q6EZey zeG{gXjlI2w!Y092Hg^0bzO2oX6&O`e=B=nK3netlz_?&kVZw6>@4~9Qacq@4FH{lU z>f(N474sn2d`sb55HKyF2Nm(-Ghu&KE|assYb z-65;}Nw@Z&thrpM>^Raf{!!J^85e@=D6yKN5JC$P{tVW6XL zyBs<=-r=qxbuubtGGGQ_&X5(Fj?Lty zh7#!{_jAoTx4(W9oOJr|JJa7x{Hk|CZNj32Mv6=GV|b-7pS_Y?n4gFzXRGldzY1{~ zgcUid=zJyqAP*SQn9T43WCrDc5v0z7_^5B75o34j^7-c)=21&tbt&vii!?-bM3$og z+fOZpcu<&Nx3?Hh*n1?D|^(u_UpBbKs(n#5<54Y4cQj@NtWRqiwyKTeTj`h1*j~-yVWav1b z{6&RMekwaDJ-@hYwpl`*B`sfF1=I3FW-)?cdH`499heg2qw0>f)NQw(^9Y_}g}4ia zcFOXy`F2G1qq|yTNc)Nntq&jJ_REs(mwn^-@~;Ob$RHne`?Htd^vS0QwYh3^%&+nn z&n)woS8$+@c(g!So1(1EOff@M!QwI=ZG0}589+=|qfyfn%S{gZ=>d0QxlEM$*la`> zW`O#KI%>9b)NPL${FF|av^zJlk_MR*F|$DHsl%#)WpFuv+U?kx2tz(W3lxUe$Pv3V ze5gv=;z*X`b~XGpA=zJ172fG685Umx4KEvXV85-7zHMmG%;Yi*9Hqr&FYcD3i95N> zBU2$cmJHhsce^`QA8vbWTkDZJSeFxNq>=d%bj*{@ixGSst4|}>;T&|1hpGPi-bCmp z=Wwg0CYO1Aa>gF^BYcBD(*q=u4#B{)hr$_RIVP>cpeN_PqmCs`(Kn{2rGByHz3z9; zk}XwKu;aOcSLP+uX3}t!?g>ou`Vtz4os-cooiniaC3yVk(d?bf3~#yD=g2QJeWkFq zb@>9sVq4VMe(&Fc(z)quY-Vm5idQ1GDGFnfvbVa^WF~4OJE~J0;-FvMQn!UdUxs^3 zlRCwVs8n8LE2efiy`RRvBXSx+e>k^nf9t75txuogI#$bQ#Fjz+WF7CT;Y(G=Y>buV zo`lCSNvDpv;kwb+Cp0?6>imFNi2)2trTEFZsB`2Q@>5Dxb)}hJ0kf+-f$DZ{_=!u0 zuy4|`z2#MYA2sR|(#qz=a6gT8avsY`gg1)O;`?LVmv`fxmuC~3+{xmk7mdrpCycKO~`1V`OCHG>c_7;bs?Pj?0u_-fyvpy!FHeZ>Jbx^PfivzRV46=)Fy=BaeHzYJVHd_%j@putpWJ03sE~YPI zf(Hu(U#aw*8FXrL0L-UR8#Q;tWYQ$OSB{5rUYReWqv13PosGul73K?gc1Eu1Oc|a^n)02x3lC32GJHVx(^Oo+9(1X;wcFaBc*+r? zJeupXB6|^faw4M@rBnCD^sL19j-I*ymvk9*o;m$;#|*eFp*DjjdjHI7KO#-PG1eSC z5$0s7C0|aSSs9vE?w#q#^ao5C7Mc0}5W+$wg9XV_KEi==;bSOB(E)R_ArJ4Qn8**w zkjQG?+SsviWn0q<8V|-(zV|npa&Rn+JRLb?MZ>ztDTKx|v`+5WxD%=|HA|`y@<(IF z8NO6C%QyX@a#^z%Vrtf}>I7<5Fo7MNR^ai@W=|VU@U%;&d3lA)@Lrh*XSXhEYCC$$ z5ptrC92z&v!3z;9m#!EZoyNN{24Tl8f7HRp@}=tFxOt`Ng+)ee)5njaa`^O3$~FC! z<>s)U^r-ZFvBN*zp+?x?#27c{`YS`_o`A}e!&MI$j8WAQqQTZFhNwZd1$~B=Gl?~9 z6hdK=vaiu;zb?i{{a?s~kJ7PVmYcEkbnG%_;y(;?%WE{dOaz_wGiYP^7cT19{&?Gw zy>Jnd(nTvN66Ib8(GL^sKg77`CiPsHFlFJcBir7XoKTy?<2-I%B<~L|*`J3^ay2H0 z`%yn^f=xMU`I(L(X0}<%YY2Jmku$OjS0S^M)}FY0AiSegPCwX*ES1YoYb9Z=A)H+v zgsHiaCn>xz{zML_)p*vRqB|Z6Aan4I03 z(RS%l5Z(z1HaPOy4DXd=1U5L!X*K*5{l(rwmY40f0=0Qqj$aruPRg~(;e%M4#F&Lc zJ+cKga*yRpwFy%XM}@kbj%k4bFHeGrd4;Nf=7oYmwLDFe*R&B!#T757NhH1RmM121 z*^W)9D=&PEO3L-~CUk(>=34f+`|Rn5u={FTvo~RE=a_-BKKYYQzgJy0I?tP?Y~n3q zu39Pjj1w`c?q^9s)(@w?@)r#% z92H<`i6}V{Q{C%=EvmY1wNH1salXR);Z%7k2c$HTQ6J!3v_c{Eb!^vhID`sB&qdtW z*O#ju&11Aa#&S~5dG5A{mSNs(oxd~2;?xIAjp)XO z%azEUV0yzF1JNGfN-{AYhU7XvGs$a~`vXW&U9ytA;hjj!9wO_5jqKx86FQivI~>y% z*`zr#+DCHbfe@DW7?*IKQY$C@toFxG#&i{&sqplgQPWu`oX^W&bU4lS`hqi2p$m#) zTISlXPpg(mp2U?ZEWn1-EH3j(SWAGJ7=m(==R#~ric!bhn6*GMbY!f#3yRCaJ7h7Uu>$r6C!HJz|#@wqIUC>$%m!a*`XThYqJ3!&gOoWrw4-DKZyz zoZ=2(N#=0IR23TGh6=1RhoinzHNH(=cqc8EpOgZu3Xw~PuC=+gQ6&6E@LRkvvcm!u7 zdui2x(}sneHaJ4*D2vWWeK9Ap1qYeZZn+bYyQ#BB7gK{`@a!Dn(`(3DIi6T{H##r6 zRhbue9+v63WZC;a-;AQq+L!&(RMi;Z^CrjUn5gdl)2k|~)73ncR_H7B1msNWN)O3P zIz*9|0X{Eu$;D;B>YK~tKvRd=6sJumVyZ^U0aY`3rY?6^YPEoU;{5gnkH?Rcu4v6V zrv+<}NeY#CDsdi+rntDvCvvqJ(|+j_XWhTLyoYqjAlI)GZ_7U#?aAXkC84=7)g3x+ zpQ>)EItz5=V+*I|16;}_M9`HQtn`HU(flVbdH%!e0xbR3QGjCIK!tpCM1;kk?2j_! z)Kq|T==NoMP{)|2!|OcxlkI`GsC0?yd=q2c`PbGc>ymJQE~Cqx6e>p(uzl$&A@}#r zo`+~jAqPYwNukN%S#Fsmw9khg4P`Tc1-QBe*UOcdv&NM zCy7#Zc1@Qd`BEo63G+iKx;!~RmcaM{2Ve-rL;(ljjn0YA%|ow~YITQsp?R^jb#S-s zU4`nltvwWDv^zRiTMg7X&EY0k8<%5|?d-JzMq8(A#s^s6F%+zlv4SYO`VVh)hA>Lo{FgJoztOwUqp zGF@3-m@bd4e6B20UE)nhc#ph>`KeR@PRGs|of?*7e;gYv&OYlAml=33a)eJgFllrg zUn-5=9lHn}`_E1%C#p*|!zG#CKu8X&eS$F|GW@$2Rfyy$H)rN}@hX6}jB=*S%rUFV zI4M_fm=WZvlsaUtr#d(?31_mjT}G_%TwY^!$edhHcux@TZj(ITIsPE*FTl?J2o|66 zToFq*xe%47vT5h!dM0k;FqQ@(8SR(S6^0{gP)V$ir|virMP2&#kM;Rvxed;=4w5oM zv+v#~#{g3Z|a;*7BKUB-Ou4^c$!zSd#6B@Z8tHrK(;?TssN z^w7S5U(@lD^3G^LxXzG)*0DH0RhrfwW14N5d{vjP782R{o=}N*WMaTGjfcD}5;#a8 zV>*Yg93-xi&K}2w9YK#DsyN}L{`m~qFJEGk^H<`+ay}Ho4b!}y&U;{WX+#<@6V%gd zHS||&jk%nbBKvth8`rk~anu!kp>5FuykL##1=jG6MoClkSFx4nt@f12UH+wYDz;?z zE&3@7mCoy&eNDVDQ16)JC3W7*R-tlf`ILn^Vd6bbeOFdRo<-nf`vF>6WWjbBHv8Dj ztk#AN=&bF>55~`9X=QKS_o02b44fN6&)pq~ltqV0Ij`W=z3i)(&5mgt{Z)B1lwrL` z&5+0KYMn4S9Y@Y)1sraDY9NpOxCGiMU6rNoVCkV6Slh}EA}|Kk(B=$vT1Lmw8(Zq< zSuNz23f5?pi$f)OqR>go0`I+0uYLnjFM1W8 zCL17dHLoqw5~r)T2XsZR4P4<`2x)3{$!wlk(Ex}gd?tY5em8o=<^8BfEN^}Y#;LRa zw2mh?biNu-c;hc{UY6SYP1P`vRb!LH!A-lR6#Vs6c<^f9I|FU*lVe@AZ2XD1_{~Xk zxE6gvzP_MBGRmj+Xy1D3?1^<}j~s75g+KGr;BAVDBpVw!dk`*%NEykFu@(B0P@o#N#-eE7Zc8zEb32whW#A6)|0++c)KC z)kL-X%}xxWgL=FrvL)ptmXU}|E=-%L2K7#&kR!}qhSo@$>d(eI3S2&S-2t-!>Nf!6dDsE-Ha5bRK5ITudg zYkWk&;TH2TK|y!{sR!wXm;%*T1$ck1!nI@j8ob;^v{~n_d^h^9hOgO2KqWVw@&N~^mDbV@Et=3KA2g3 zIzGx{-jCqBH2Je$MIkyoGQzLy8RI=|_dMynY#v+E69W~d8ZKRYO(Xk&3Up(c8QzIp z!~xeGF@TRM{@1z;#X7${kyXOWMr((ag}y02v!E|eVw+r<8ao~FmooR_GpYw@JRC4C zre`|4+@gABih7aaPQe#5_(~ukk8)forjIX!?i9Rh#x-v|Qn0?G3|gMaZys9e@zD%c z#(S-?+&S<@8evw&4Bz;$r{h@c@SPPkxbv08J?d;N86S6Hhl%$m@bg@d8Wv~b(8ZJ!5+#E&DPICpQhv~~6uBx%XU#DBKq%ysWci zAe0?>qIpT==rv($hYuB+YhsK`>Ur@um%q{^jq5X=e$mBZ=Oba7qk8x2pJ#jK;LNY& z+G2l2$7FoKfJ2WWeE(SCsVb`Q% Date: Fri, 8 Oct 2021 21:09:17 +0200 Subject: [PATCH 06/14] adjust docs tests to use test database from config directory --- docs/build.gradle | 4 +- .../ingest/common-log-format-example.asciidoc | 24 ++++----- .../ingest/processors/geoip.asciidoc | 46 ++++++++++-------- docs/src/test/resources/GeoLite2-City.mmdb | Bin 0 -> 20809 bytes docs/src/test/resources/GeoLite2-Country.mmdb | Bin 0 -> 17952 bytes 5 files changed, 41 insertions(+), 33 deletions(-) create mode 100644 docs/src/test/resources/GeoLite2-City.mmdb create mode 100644 docs/src/test/resources/GeoLite2-Country.mmdb diff --git a/docs/build.gradle b/docs/build.gradle index 5ecaf7dc6faff..9540b840e7943 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -64,7 +64,6 @@ testClusters.matching { it.name == "integTest"}.configureEach { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' setting 'ingest.geoip.downloader.enabled', 'false' - systemProperty 'es.geoip_v2_feature_flag_enabled', 'true' keystorePassword 'keystore-password' } @@ -89,6 +88,9 @@ testClusters.matching { it.name == "integTest"}.configureEach { // Whitelist reindexing from the local node so we can test it. setting 'reindex.remote.whitelist', '127.0.0.1:*' + extraConfigFile 'ingest-geoip/GeoLite2-City.mmdb', file("${project.projectDir}/src/test/resources/GeoLite2-City.mmdb") + extraConfigFile 'ingest-geoip/GeoLite2-Country.mmdb', file("${project.projectDir}/src/test/resources/GeoLite2-Country.mmdb") + // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 systemProperty 'es.transport.cname_in_publish_address', 'true' diff --git a/docs/reference/ingest/common-log-format-example.asciidoc b/docs/reference/ingest/common-log-format-example.asciidoc index 05f012ab53448..2d5391d09fb88 100644 --- a/docs/reference/ingest/common-log-format-example.asciidoc +++ b/docs/reference/ingest/common-log-format-example.asciidoc @@ -175,7 +175,7 @@ PUT _index_template/my-data-stream-template ---- POST my-data-stream/_doc?pipeline=my-pipeline { - "message": "212.87.37.154 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"" + "message": "89.160.20.128 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"" } ---- // TEST[s/my-pipeline/my-pipeline&refresh=wait_for/] @@ -216,21 +216,21 @@ The API returns: "version": "1.1" }, "source": { - "ip": "212.87.37.154", + "ip": "89.160.20.128", "geo": { - "continent_name": "Europe", - "region_iso_code": "DE-BE", - "city_name": "Berlin", - "country_iso_code": "DE", - "country_name": "Germany", - "region_name": "Land Berlin", - "location": { - "lon": 13.4978, - "lat": 52.411 + "continent_name" : "Europe", + "country_name" : "Sweden", + "country_iso_code" : "SE", + "city_name" : "Linköping", + "region_iso_code" : "SE-E", + "region_name" : "Östergötland County", + "location" : { + "lon" : 15.6167, + "lat" : 58.4167 } } }, - "message": "212.87.37.154 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"", + "message": "89.160.20.128 - - [05/May/2099:16:21:15 +0000] \"GET /favicon.ico HTTP/1.1\" 200 3638 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"", "url": { "original": "/favicon.ico" }, diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index 936286ee5b2fc..7ee81e4148a75 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -69,7 +69,7 @@ PUT _ingest/pipeline/geoip } PUT my-index-000001/_doc/my_id?pipeline=geoip { - "ip": "8.8.8.8" + "ip": "89.160.20.128" } GET my-index-000001/_doc/my_id -------------------------------------------------- @@ -86,12 +86,15 @@ Which returns: "_seq_no": 55, "_primary_term": 1, "_source": { - "ip": "8.8.8.8", + "ip": "89.160.20.128", "geoip": { - "continent_name": "North America", - "country_name": "United States", - "country_iso_code": "US", - "location": { "lat": 37.751, "lon": -97.822 } + "continent_name": "Europe", + "country_name": "Sweden", + "country_iso_code": "SE", + "city_name" : "Linköping", + "region_iso_code" : "SE-E", + "region_name" : "Östergötland County", + "location": { "lat": 58.4167, "lon": 15.6167 } } } } @@ -119,7 +122,7 @@ PUT _ingest/pipeline/geoip } PUT my-index-000001/_doc/my_id?pipeline=geoip { - "ip": "8.8.8.8" + "ip": "89.160.20.128" } GET my-index-000001/_doc/my_id -------------------------------------------------- @@ -136,11 +139,11 @@ returns this: "_seq_no": 65, "_primary_term": 1, "_source": { - "ip": "8.8.8.8", + "ip": "89.160.20.128", "geo": { - "continent_name": "North America", - "country_name": "United States", - "country_iso_code": "US" + "continent_name": "Europe", + "country_name": "Sweden", + "country_iso_code": "SE" } } } @@ -236,7 +239,7 @@ PUT _ingest/pipeline/geoip PUT my_ip_locations/_doc/1?refresh=true&pipeline=geoip { - "ip": "8.8.8.8" + "ip": "89.160.20.128" } GET /my_ip_locations/_search @@ -250,8 +253,8 @@ GET /my_ip_locations/_search "geo_distance": { "distance": "1m", "geoip.location": { - "lon": -97.822, - "lat": 37.751 + "lon": 15.6167, + "lat": 58.4167 } } } @@ -285,15 +288,18 @@ GET /my_ip_locations/_search "_score" : 1.0, "_source" : { "geoip" : { - "continent_name" : "North America", - "country_name" : "United States", - "country_iso_code" : "US", + "continent_name" : "Europe", + "country_name" : "Sweden", + "country_iso_code" : "SE", + "city_name" : "Linköping", + "region_iso_code" : "SE-E", + "region_name" : "Östergötland County", "location" : { - "lon" : -97.822, - "lat" : 37.751 + "lon" : 15.6167, + "lat" : 58.4167 } }, - "ip" : "8.8.8.8" + "ip" : "89.160.20.128" } } ] diff --git a/docs/src/test/resources/GeoLite2-City.mmdb b/docs/src/test/resources/GeoLite2-City.mmdb new file mode 100644 index 0000000000000000000000000000000000000000..0809201619b5915fa6189200e4c53e6088f437da GIT binary patch literal 20809 zcmZ{q349bq_Qz|wdjL^XKu|G-C|o2F8tyn;17UZeiTYxQ%f;;||81jJp_jGsZF!8A*&}MhYX9k;ZT_ z(iv_>1|yS^#mHvlFvc-*8TX+1e=kHHr}7yEj6%kE#stPh#=VS5jLD2CjH!(K7)6Zx z86F|TyFyF@YK16fVF|;8@IfJ_bE*`nGKyju$zB%DAV`6jNqM52QNi#r{6bVlOU`0p zK!{*eSS3V=@A&{@He(K>nm`Yl%j&-q0*-=&gnuB|8-@5|R9M48SXT!FUlpPrK4m*)M6o)L{m#yZy95`5Gz>y(P)vCoVSXxT8K52*9|w4 zmOloQ4hr!&r`ARF)(i0j3!h|cU~FV;Vl)uQ1)GJ~!osal;Wi<*vkH1(noFL9ASF3c$4)ajONZfJdB?AHsddhcL*}$J>2ZSEc~kw?|15nzX|bo{B7y>1N^_ql58Jf-Hsq65?M%e8zeIW_%va`$C8>S@;#> zKa8)VMZV#?d@ICvocBHBhiH)>xyVn19%F>~8DkcD<1YZ5{40g_d*s@ue6BUkQ zyXM2!~{E<<^sAbeK>KP9)9%k_TXM2QG^BD^m3kl+WVOz{5EnzHW zEMqKZtYAFKSjkw$Sj||&;QnWOj8l&@)-l#Io?!6&XXE+LwvmOK7!8cgj4h05~07)KeH`_N*x}x{>*rl@fzcG#v6<`ahJ*5(h*^6 zX5m|mw;6w7yn~Xt!uBqw-b1%PBW!==)cXv~ebT_cGd^H^$oPoy4+6r8N`B1v1h>9K z*jjMUZ-nhE(7`6PGTMZ#olBl0Tnt;}gZ_z<{}#4SIrT5bXM~=r7k`c-UkTe6ocCp1 z-hWv5HKEsCocb2F*oEyo==~&Y-?Q)s#*YN+>HkGZP1t@$$zMgU2=EI5mT3ZcHY+9b zq)=xVv3JqBpkxoBbp^T$ts56PFRsY>EW992cp(&e3hg4!yO^LVt@UDI?>L1%P`E*8 zeSxclb}2AGXqN%~h1QQ1E|0s*6)e1xpxmPkgu)=9IXEvNPGK+$hY*yPw4qQKDYRk0 z)k1S}k>PPgu3_PbIN`NWxL#2?<05i3Od2<C(DKr0YhVN^Jtg%cPPg?2CBY*J?tsmpy#o7ySV?h{%OmsEKk zPN_N+bBcS8#`7N;Fq7ej^iBxvLEuTD{SjCy zv>MscLn@_7$4<`Mc{&#C!PSR}LsoLb0v%4dsN$n)Q&8mE>)VU5t1 z11p8Lf`yL~E?vVZp8vGfl(K3o!)=|=9^)d96E0JM!tR<}w0EfFfD%@CkMUQ6Rh7R%;X|SE{HJ|Dd6r*3VugPY25>NFA4B15p?v~$2(1NZ z6WUoW*-EfXYG>g&f^|>Kf7+Kq`xN-M(Ei0mc>c3Y`kaMd5UiVh#lrs(29Dv>H@M4x zh4wA*gV4TX;r9ef??)E?MChaX(a&_Ssr|xv9B!(?bsGtF?kmb_-GD-}(Cxqwp?3lL z2)!$Cq0qYl=L`Kj*6U6f$P=`F0SkK&RR7a2frNpP@|1r2!$y^9}nCs^a)&KB4H3ipTxq+1j~_Ap>V&@@8i58 zg4NO<7EU8rl|~QOgF-ifa-mNLyh1PKl4WrvG5=w})n^jaT%}h)p;G8R&hy79%wl1H zFj&QB6%>9a^bjyd=nrs_*#xT}@%*RHC86a(Isg4p=zm~^2MJcCxo_&Vad*M|XFMYG zhXC}khoSzm(B}c$g#HMyM(Fc_WkO#7EEYP?fBGW2i`9LX@bi`u1|R3tawx16`U*}x zN*IEBO82Z{tS0nPmaT=t2B9OM_4PvM`A=U*MXWmT{HH%jLN%G{8==*hDV6V`30#6Bj7c1;0SeQ&j}qq(@zQg1Pf0R`u)nO(~M^c)|mA?6kZg1nDbsBT&C{% z5)1!Cu-fPqD4Y}epMm#<{wna6&|d@I6#DC|@CL!^2oV-G6Ri5Z4TX1w{uj=BhhTM* z_gMH>g5~SKLE&Se{~h>9=pS&A4+)lK|6t)6!q7`O^$8SOh2FxcvjnT|Z7ggjSWoYO z!hePSPv9G&e+qmh^nU@L3;i=z_&33N>=!KjGVU(_Vd2+tCBMa8eh~V1ocBFpsA`oT zS@;vdvgK!VCqwAJ05+j>Ga7=53|mVuG=@%4-Zkt{=q8LVoY$40y0&p13%e6k=o=S6 zAzc_ffZ@Wp5V%|znE#Bv!nhddEsUP5*Nb31=n@w8;hQO2*z?9^oX7K@s*%wjHyb33 zD}aH*xRQ$uAPmRT5y^&wkw8#0i7^-oLxnMf^R6POQPCL2LMK5*opCi3ZWYEgz-VEN z07eO8B$vFF(5Hq|*D^gkVL_bQYEp)Np8cp)f}nGk{sbmXggNoZ9J^PdqCMvxV% z2x<&5c>XhHlh6tj%zs9WFy;b(5XSFV@Am}jo;?2L3T= z;4xvW;Ua4ZmX~<`GuDyNin=Gb^NzFd1i@;*Q&4zO7^i{fh4CyGd5&PkZkUBH5Ui)a1cg_G@h8rEnP7SF&n$eE zV2uo~L*WZyya9YDj5mRIg%JVX7Dh8GaCf>|)%Gtee1~A&`aLN8O&EXWy!Q!~oquQH z2L!d%Gd_YshcNyDd?Ji9T;yYd<$@OEwF%=a=d}{7=e4u&T%5u`q41e7KIOcB5w7{1 zQ~zdg%IaKSLI|C&@c##4VE!|{6~@;^%te(haY*p;B#$bKGD7YKWI&O4u=#sYf}7G6lO6fQ;zuaSBJ z*9m(sAVJuB19+Wx32>>f_W|&dr!PTSZNH3#{Rpb=_Wno>5cVrLkLN!XZuWsJbP%kk z4?=3Vunz`?3i}W)auq@K2m3Gp7*Sa>(Vs#qdY>B61_qzZd7 z7fB&ldTA_l5mdC=-AH8#dj{ua64Zj#p3TA>g7u(Wr2N8u4{*P*=K=Q$dp>}p&H@1S zD;V9;`hrx%UPZ8ikLN%8Y!X^_RwMO$ zVV}zizav=w`~wRgBv=YHNa1x}Er3^jbzG#LVCku<%p>7QUM1M)Beh7_7qG%YR^axt zWB!xvu!L|ecNqwToxgvvFPGWEzJl>6gTH^V2oFd=S=qpO5DYMUzOKq`l?z|iv1;KQfi^2Yq%&d!qN9! zQi++C8VdL;Jy75)Gsz4(TuIXdj(d`3c*eEV?`Ww%(Ne#srM|wUeno0!)ySkg0tl)bD`2#1FGF^ZeDGP=z@X z0vMRpQn$6G?&+5LT`hG-9U;d#ST-f{Xylp5@yLdbz*YQ|0yh&W`gG7l#6#;_9M+r^FiyHk727Bd01WTJ_m%j4X_fw9m8bk;$C%`C6R*z;n5Eue6)Ic;Cy-yx!Y|i$ z-(KJi7I}R|ru4t*@fC%9Fvix^u`?+xkNuR97V!9HTCPcxsbbkaF1qr1;EseZ;e^7L z`qeGPB4R2jwCZMGg#&gm@s3i7Fi1`5HXR1uw_x?s3Wo>@(fHn);u4E)i{=9 zo!fT$?1{at4=?JdZ)kNmTO4DO67!{9fos?;sfj_a>BId~aDm7T4%T6gOqg(nyQOY( zOWhItb%avGwX#GYlpa|gIjkyv_SiZEsbifqxc8Np!{12-sxKs!g{nOA&V%!D+q9%m zRk*>2dm}l+lh;zeM)rle({ObA(naKADzn9pM`T1+G}m*SA;!)gS#tKs(S)&Q6EZey zeG{gXjlI2w!Y092Hg^0bzO2oX6&O`e=B=nK3netlz_?&kVZw6>@4~9Qacq@4FH{lU z>f(N474sn2d`sb55HKyF2Nm(-Ghu&KE|assYb z-65;}Nw@Z&thrpM>^Raf{!!J^85e@=D6yKN5JC$P{tVW6XL zyBs<=-r=qxbuubtGGGQ_&X5(Fj?Lty zh7#!{_jAoTx4(W9oOJr|JJa7x{Hk|CZNj32Mv6=GV|b-7pS_Y?n4gFzXRGldzY1{~ zgcUid=zJyqAP*SQn9T43WCrDc5v0z7_^5B75o34j^7-c)=21&tbt&vii!?-bM3$og z+fOZpcu<&Nx3?Hh*n1?D|^(u_UpBbKs(n#5<54Y4cQj@NtWRqiwyKTeTj`h1*j~-yVWav1b z{6&RMekwaDJ-@hYwpl`*B`sfF1=I3FW-)?cdH`499heg2qw0>f)NQw(^9Y_}g}4ia zcFOXy`F2G1qq|yTNc)Nntq&jJ_REs(mwn^-@~;Ob$RHne`?Htd^vS0QwYh3^%&+nn z&n)woS8$+@c(g!So1(1EOff@M!QwI=ZG0}589+=|qfyfn%S{gZ=>d0QxlEM$*la`> zW`O#KI%>9b)NPL${FF|av^zJlk_MR*F|$DHsl%#)WpFuv+U?kx2tz(W3lxUe$Pv3V ze5gv=;z*X`b~XGpA=zJ172fG685Umx4KEvXV85-7zHMmG%;Yi*9Hqr&FYcD3i95N> zBU2$cmJHhsce^`QA8vbWTkDZJSeFxNq>=d%bj*{@ixGSst4|}>;T&|1hpGPi-bCmp z=Wwg0CYO1Aa>gF^BYcBD(*q=u4#B{)hr$_RIVP>cpeN_PqmCs`(Kn{2rGByHz3z9; zk}XwKu;aOcSLP+uX3}t!?g>ou`Vtz4os-cooiniaC3yVk(d?bf3~#yD=g2QJeWkFq zb@>9sVq4VMe(&Fc(z)quY-Vm5idQ1GDGFnfvbVa^WF~4OJE~J0;-FvMQn!UdUxs^3 zlRCwVs8n8LE2efiy`RRvBXSx+e>k^nf9t75txuogI#$bQ#Fjz+WF7CT;Y(G=Y>buV zo`lCSNvDpv;kwb+Cp0?6>imFNi2)2trTEFZsB`2Q@>5Dxb)}hJ0kf+-f$DZ{_=!u0 zuy4|`z2#MYA2sR|(#qz=a6gT8avsY`gg1)O;`?LVmv`fxmuC~3+{xmk7mdrpCycKO~`1V`OCHG>c_7;bs?Pj?0u_-fyvpy!FHeZ>Jbx^PfivzRV46=)Fy=BaeHzYJVHd_%j@putpWJ03sE~YPI zf(Hu(U#aw*8FXrL0L-UR8#Q;tWYQ$OSB{5rUYReWqv13PosGul73K?gc1Eu1Oc|a^n)02x3lC32GJHVx(^Oo+9(1X;wcFaBc*+r? zJeupXB6|^faw4M@rBnCD^sL19j-I*ymvk9*o;m$;#|*eFp*DjjdjHI7KO#-PG1eSC z5$0s7C0|aSSs9vE?w#q#^ao5C7Mc0}5W+$wg9XV_KEi==;bSOB(E)R_ArJ4Qn8**w zkjQG?+SsviWn0q<8V|-(zV|npa&Rn+JRLb?MZ>ztDTKx|v`+5WxD%=|HA|`y@<(IF z8NO6C%QyX@a#^z%Vrtf}>I7<5Fo7MNR^ai@W=|VU@U%;&d3lA)@Lrh*XSXhEYCC$$ z5ptrC92z&v!3z;9m#!EZoyNN{24Tl8f7HRp@}=tFxOt`Ng+)ee)5njaa`^O3$~FC! z<>s)U^r-ZFvBN*zp+?x?#27c{`YS`_o`A}e!&MI$j8WAQqQTZFhNwZd1$~B=Gl?~9 z6hdK=vaiu;zb?i{{a?s~kJ7PVmYcEkbnG%_;y(;?%WE{dOaz_wGiYP^7cT19{&?Gw zy>Jnd(nTvN66Ib8(GL^sKg77`CiPsHFlFJcBir7XoKTy?<2-I%B<~L|*`J3^ay2H0 z`%yn^f=xMU`I(L(X0}<%YY2Jmku$OjS0S^M)}FY0AiSegPCwX*ES1YoYb9Z=A)H+v zgsHiaCn>xz{zML_)p*vRqB|Z6Aan4I03 z(RS%l5Z(z1HaPOy4DXd=1U5L!X*K*5{l(rwmY40f0=0Qqj$aruPRg~(;e%M4#F&Lc zJ+cKga*yRpwFy%XM}@kbj%k4bFHeGrd4;Nf=7oYmwLDFe*R&B!#T757NhH1RmM121 z*^W)9D=&PEO3L-~CUk(>=34f+`|Rn5u={FTvo~RE=a_-BKKYYQzgJy0I?tP?Y~n3q zu39Pjj1w`c?q^9s)(@w?@)r#% z92H<`i6}V{Q{C%=EvmY1wNH1salXR);Z%7k2c$HTQ6J!3v_c{Eb!^vhID`sB&qdtW z*O#ju&11Aa#&S~5dG5A{mSNs(oxd~2;?xIAjp)XO z%azEUV0yzF1JNGfN-{AYhU7XvGs$a~`vXW&U9ytA;hjj!9wO_5jqKx86FQivI~>y% z*`zr#+DCHbfe@DW7?*IKQY$C@toFxG#&i{&sqplgQPWu`oX^W&bU4lS`hqi2p$m#) zTISlXPpg(mp2U?ZEWn1-EH3j(SWAGJ7=m(==R#~ric!bhn6*GMbY!f#3yRCaJ7h7Uu>$r6C!HJz|#@wqIUC>$%m!a*`XThYqJ3!&gOoWrw4-DKZyz zoZ=2(N#=0IR23TGh6=1RhoinzHNH(=cqc8EpOgZu3Xw~PuC=+gQ6&6E@LRkvvcm!u7 zdui2x(}sneHaJ4*D2vWWeK9Ap1qYeZZn+bYyQ#BB7gK{`@a!Dn(`(3DIi6T{H##r6 zRhbue9+v63WZC;a-;AQq+L!&(RMi;Z^CrjUn5gdl)2k|~)73ncR_H7B1msNWN)O3P zIz*9|0X{Eu$;D;B>YK~tKvRd=6sJumVyZ^U0aY`3rY?6^YPEoU;{5gnkH?Rcu4v6V zrv+<}NeY#CDsdi+rntDvCvvqJ(|+j_XWhTLyoYqjAlI)GZ_7U#?aAXkC84=7)g3x+ zpQ>)EItz5=V+*I|16;}_M9`HQtn`HU(flVbdH%!e0xbR3QGjCIK!tpCM1;kk?2j_! z)Kq|T==NoMP{)|2!|OcxlkI`GsC0?yd=q2c`PbGc>ymJQE~Cqx6e>p(uzl$&A@}#r zo`+~jAqPYwNukN%S#Fsmw9khg4P`Tc1-QBe*UOcdv&NM zCy7#Zc1@Qd`BEo63G+iKx;!~RmcaM{2Ve-rL;(ljjn0YA%|ow~YITQsp?R^jb#S-s zU4`nltvwWDv^zRiTMg7X&EY0k8<%5|?d-JzMq8(A#s^s6F%+zlv4SYO`VVh)hA>Lo{FgJoztOwUqp zGF@3-m@bd4e6B20UE)nhc#ph>`KeR@PRGs|of?*7e;gYv&OYlAml=33a)eJgFllrg zUn-5=9lHn}`_E1%C#p*|!zG#CKu8X&eS$F|GW@$2Rfyy$H)rN}@hX6}jB=*S%rUFV zI4M_fm=WZvlsaUtr#d(?31_mjT}G_%TwY^!$edhHcux@TZj(ITIsPE*FTl?J2o|66 zToFq*xe%47vT5h!dM0k;FqQ@(8SR(S6^0{gP)V$ir|virMP2&#kM;Rvxed;=4w5oM zv+v#~#{g3Z|a;*7BKUB-Ou4^c$!zSd#6B@Z8tHrK(;?TssN z^w7S5U(@lD^3G^LxXzG)*0DH0RhrfwW14N5d{vjP782R{o=}N*WMaTGjfcD}5;#a8 zV>*Yg93-xi&K}2w9YK#DsyN}L{`m~qFJEGk^H<`+ay}Ho4b!}y&U;{WX+#<@6V%gd zHS||&jk%nbBKvth8`rk~anu!kp>5FuykL##1=jG6MoClkSFx4nt@f12UH+wYDz;?z zE&3@7mCoy&eNDVDQ16)JC3W7*R-tlf`ILn^Vd6bbeOFdRo<-nf`vF>6WWjbBHv8Dj ztk#AN=&bF>55~`9X=QKS_o02b44fN6&)pq~ltqV0Ij`W=z3i)(&5mgt{Z)B1lwrL` z&5+0KYMn4S9Y@Y)1sraDY9NpOxCGiMU6rNoVCkV6Slh}EA}|Kk(B=$vT1Lmw8(Zq< zSuNz23f5?pi$f)OqR>go0`I+0uYLnjFM1W8 zCL17dHLoqw5~r)T2XsZR4P4<`2x)3{$!wlk(Ex}gd?tY5em8o=<^8BfEN^}Y#;LRa zw2mh?biNu-c;hc{UY6SYP1P`vRb!LH!A-lR6#Vs6c<^f9I|FU*lVe@AZ2XD1_{~Xk zxE6gvzP_MBGRmj+Xy1D3?1^<}j~s75g+KGr;BAVDBpVw!dk`*%NEykFu@(B0P@o#N#-eE7Zc8zEb32whW#A6)|0++c)KC z)kL-X%}xxWgL=FrvL)ptmXU}|E=-%L2K7#&kR!}qhSo@$>d(eI3S2&S-2t-!>Nf!6dDsE-Ha5bRK5ITudg zYkWk&;TH2TK|y!{sR!wXm;%*T1$ck1!nI@j8ob;^v{~n_d^h^9hOgO2KqWVw@&N~^mDbV@Et=3KA2g3 zIzGx{-jCqBH2Je$MIkyoGQzLy8RI=|_dMynY#v+E69W~d8ZKRYO(Xk&3Up(c8QzIp z!~xeGF@TRM{@1z;#X7${kyXOWMr((ag}y02v!E|eVw+r<8ao~FmooR_GpYw@JRC4C zre`|4+@gABih7aaPQe#5_(~ukk8)forjIX!?i9Rh#x-v|Qn0?G3|gMaZys9e@zD%c z#(S-?+&S<@8evw&4Bz;$r{h@c@SPPkxbv08J?d;N86S6Hhl%$m@bg@d8Wv~b(8ZJ!5+#E&DPICpQhv~~6uBx%XU#DBKq%ysWci zAe0?>qIpT==rv($hYuB+YhsK`>Ur@um%q{^jq5X=e$mBZ=Oba7qk8x2pJ#jK;LNY& z+G2l2$7FoKfJ2WWeE(SCsVb`Q%OB zC*prq!~%|q*do4!jCvG5aByJ*ZCVoiVLflH+M%+%^LEK4X5!u8LVknVA}w(mI)*YBED)CWTL>gtMqHo0uaq zmrdrSnmjEspTY&iLShl|46#^o8fMm5z7(_cXOU&Viz3S@d{&_Ydyf6B5P6<;D^qo= zL{_ItYed$jO6#b%p4cD)2Qn|*lxnh>buSQGM7C<(=|5%5Z7}H#k(XH7p3-|+WCw*i ziCx5QVh^!bp<{SOG~#pxbbX!mExs;9SE3uyU154KEqkJ)4dOc! zXb|67z}@2O1>7mVv)SYv;#>%C7T}!V8ECiHnHd#KpuV#HB=l$RPRMO)|IZJd*Uw@XE|9k_qSzR&mX3Qw}|i7ROvSH-A=tb+J!L7Smw(TAAQ0%giVGLIjAdOlU(uTQONw4HZb2X z@fEVJNTKf>PQ4N08<`RoQ+Sv7N~lnpsw)#;d3y=3@ZHm1@{JPTy|5)j_4|m?;=7-3 z8ACjfQm7E$gA@i+!m-q=B&wtfLy2!3TaG8H3Fbc^^Pg`bg*8Mi5mD%9qkLx_@sRjp zY!WBx+gqZwDm*N{NvwN>n2hnwXWbO>J=(6|drW+fv&j?0lf+bF8quiGS2u}oda5)- zd^6ePsg#iU&o^6q$bC@q%}v$KqY+Oly3Q2ef>hl?@hzglGsI$I39(e6Ml2KGatfbK z37P+VE5!Fa>sAu0h}FazVlAe&CSD^F#OnmoBgW|a zDe(s3IxX{`@8=Z0MZ8TQ_ZdCrKOgg-kNMAcoOQn z{fT&=_%rbr;sfGC;;%$Aae`(?W;4`|Gk3i9sY{DV@ zL?_}j1>8wnoa_& zZ8S^gp~;0}ozKz*9qKNku(zT+j4;Mu0);_hT?+ISD?nie(MRF+ei@ovA=c$Ue=+9? ztDnNWB@?9qUdcUcSXc5b13SEBFj`(O)>Xi@Vln?&%zs`-KcL=q9rTd@teeHUk##pI zyg@VnS+{ghxD5*Ti*-9NT&z2QT(Ry1hKQBLw_pq~-h0bX3Ud_R^yEQdm{|F&E9jt5 zNMVt}GkydV%EcN9l!#T#CUL$hJhO>TtX}*mJ!RvdKOJq zvhBw1YZN`)m0m~TdJ2EZ(nj=trC6JwcS5Yqz)!?_0eDrc zEx-=3wgNAS^&%CvQJwMH+D_riik|Ld?S#S}v39X;H|x04)?Nyk|9YNaX&)4hinSj& zB-Q~64=T>2e_DqrJfiSMYZmo2v3|t5A5)>3r3CRhg`cv7{Ac|}tT%vn#Cj8WORS%< z?&pfLT$g&A!e1zQ^<)WiX8lU6BumFx$Dvrirtn?G*$piH77Fi+g~hS{Al7>n{$6p; zIF|lMa6WsutMX?kd??mmSoeX#d*@#%ib&5^? zra0#dmOdu_uJ9J_6Da&kEO)a0sdZ<%^Z6O-{w>z$)Z<&c$@-GQ|8!9J8YcO~`UZO6 ziS=JL`Bri6be8_7utnhx#kQd57aLM+8{1L1^I~_R@HEA_+<5jGQ0OgoXW&e+y8zwA z?#h}x2*%yf23piKov)SYvg=f-v6rQi}UUngc7b(s|@5bDV@s`WP zz68h+`%(%63Qw;Og?$x0xbF7lP{PtHRxQ_U%x}5&I4x zTkJd8B#U|vvNVJksyOc@mU5v`Aa)*0`HJ&tpgoMj!VbdWP`F=gxL&VPj^Mn!wHmb{r51BEKF9{_@4SFj~Vi#?8Y;}zaJ!xT>FAgqDHT(N6`$Ha~R4Pr-unAmkxc&LLyoWgpAH)jY|_GGaq zvF?!$3R5V2RN+P&`*A2t7yAidn%GaW$yCJ!Tn@XD!Y1l5zSuLMFiY&2tb0n)(_LwI z8_uEd1e?r*!t-K34J;9RKCnpa1*}`xq2)6aE>^g?%3cbEXT@H|y5$`do}+Mu;sS(N zv+-6!VT0JKfVE<;rjS1G?IY$td%f0qGq@27FNnQ~3Y!(4_iUkXtHOH=@}ILqY&f)o zJ+>X{pNjo5@C&hb07u2%3G5Sl7qD0C-F(X)g%@;OAAD7X7nZWgekdFgoB7W^$huEh zcbGV$@S?^sDEw6H9{~xmng8t9w23!{*Qxgtg_}(6H=yuyvEO9f&pIf)Md8~D&$4%* z@S)hh1pXj)68N3i$ANdn{uLE|t?)M2Zz$yAT;%Sf_b6ok^S09;@s>Y}?Z(0PwTV~v z7b<)};ioM96$+=sZU$P#cHON->)bqTx3MLC*Na(ygTmj%{)lxSE6#M^`40*?*WMca z6HDAv?0*41u|I?Q*J6JTd@1%9RQR`|r@ImUL*Z8nZw%i+;XASa%err=;9C7ZEjgla z=i0H*-x=cAKqqlf3HNsV3fD5{Gzw2w^d7)cXDDh8aIV7Z_IwI2=gq#1kw$2c|Y?wGhfjn_? z*d$kR$+Ilw69o!4lQ@M?7$Hs(>xL`bsOXHOkQ3s1ol^pZpg5(#ed3e>_lQ%@mUmN~ zo68wR;k^nsKscBe=K*op&j8Pfvlv(=&Jwm;`@#&K}^W;_L+uiSr7uUz}H|zE9B$Li4f%1oL0dT9yt&AtBBY;78&drSO=-nbxg?rX;{sM)+it_>MK2&)9HB-oi^L(Ne z3V#=;4fvZl?i8HTmY(l^M7@s{Uhn^a!av3Ngms@PynXvG3O}QQ8%DoC>3nhi4fw_R z66)WH^B>?FalWF$*NRJFl3CRM65lGkSN{(s{M08Z^!pU9oqn4_N8x_z`8%P6Tg}se z&f-6vP0mobQ{eA{Qg`upWnDLg+n>J&g*_FX!dWPtBmQ2jJ6jQ;@&0ouJWt_nF8>86 zWs3hoAVd5Y0hfrsw>I%#tngZ1N@1Xbun$U?i@z`HE>n26^rNtU2jKvet``3lz##Eo z$tD98?&;S*n8K?RZlLyGgVJ^4zm|1BQ1pC|rR#|s6rRFOC`}Uo&A=$}{}8}WcDDf8 z;=dKRL;SY^__^eEg=h7h6lN*Bu@6BhSNua+m!ojq&7VhMK818C|1gy95`Q5uLi|N+ zGF;*A2meS4ixuuohQ9=*a`Bh4u8ehbegEAQ-qS&NFG|(ozYhqC-}L@~`0v*y{xJ$S zLi;Ny|3o&aVUuQ-B1BZ-dFw+c z)r&vIy12qUu=*P)e3(MFqeoC$F8;{?ewLd8OcVd30IoV81F&$9D?BfFlESHqjF~Jo z0yD(l#L{$yw<+BwPpQzeb2du2&YS~0E&jP$=bxwW=4C#G3n*lK@h?INSDnwWZn45M zVkw2o6z)fb|5=pqv)psQTJf&{R*C<400Ud8@SKnN&%Z{6o}KGZ+9>|@RM?>K=5rH; zn-!kI7L@RF-c|rV^}Wa@+Z3MOb_!os^y$LVPGFDtcd@iv;mri{pE36pMf#7B`;1)t z`-ua@LE;c`So}w*Zdx9t5RV=IYc8maM(V?nP^7+jXl1l2l$}#PYSL3e17{~PuV0fN zjz=rhjJ)hXU(@G3cZXu3a3nmw9(@%iUd0p2mDMm}M0q4!AF2uzh9l#vqO~xlumnw` z0m!SOFsra6Iyu--8yW-w3@mDH+}Pasa&yy*&5Z{FqXX~2vipz?A#I#U|4P}7@6qVoNG#zX5#YnDIA2I8FjF(yt!#ZVKR3oUxAfJk~=Yh$%V~tdr|z z>fH6&gjvglRa=q^v6$!?;*5TM9zDSs%PQ3wyOQ1KWW~dw2n-mC2Qx)|&p}tF5JDdX zW(FGf<8NSePGXvA5o^fD0Qb3TapK^L_Hp&P_(F^;yNoZ+&8}{!51KdkH7lQ+-B6!c z7s0zxoDeK&ZdzvcP2(}FecSxGYPq)A5JeZml8cTv(I+v46Z@Y&vHw6uKjVH~uPV>D zGou_Gm5gLZ1tkr&iH#ACBCjM`8;s~c@*xjK65EZl<(EWFLC1tGyG^(3>gJ}oIx^JO zCSEjJ#@*H)Z%Q6MKG)j~MQulxpW6CT%g)0XYi@lk7_N_hSb`l8j6@@oYoiVEin>_z zk;xh5@W?=yz|_DjGudvzn4G!lJJ;mW9GaY8T3J0QRHgHtTk1Z=b4wdSl`vvl3{QzI zRc4+>VP7;iZff@EwHNi*~mM1SS6vNz1#vZBV(*q&gfid-cBJP_(Wl)IT053Dq@> ztqE5KqT>Q?ZG04jiGxNO3=~J}8ftsY9j10{J$T;YcArz$pyPxj?Qg)`DjMJeo`6>)x|<_l?Q9GsuOQI34Y{KrFFZCDtbqas z((iqkk3TT`ecg4~%Vy&t;%?Kuhp>hkWAI{_X;Q(?1j!%V=Z=n}&!g_~ZM=!&Q z-DMjVtkccie~8)KC?)oo&7Fg%XvC~V>+(&lD_;tX);itPMadnQjHBMh>R*-7um3ga zJ=j_3q0mhsBV0F_7$}U!LdHW1OQQ{l=58@wHpIvW8h7zL16yWXY&hhJts2Tp8%!iG zYT5Y0sWppRcP>)rxLfPP-_tbiTb_J5`KlMhS0s;OHeS($cxug7sKVUEY35!~e;DSL z-NOmVEenPx>5ODb8Kxx9lo{~wnAl^6U*=gNe^m+-p$uDF#`T5bv_=)(3`6w{Ym=05b<{KEh`HNmoZGNtG{|?j38`{fa zopqD!k0ke)UOq6rY(+07U>P5}>Z`TrWrQ12^N|B5g~l^JndhL14taQL2#wcxipPf& zTgIDRha5g%-N}toUNq12-H6VMldE(hksyJRnCikf;V-K2XEjV## z)5+PpTV^dcoj&7^JLBezei=8XO+x0vMp!V!jR|=gq{CCx^CpGsr-WjRnDLy_hVT?O zPKV=hGYKO#PUEC?fB_Om7=-8TJi?SPJSPv}2!oB~&VmWj8@9n9>ulGC%O+;@yKG|m zl+=C(gR(t?vV+w~KPEtBTKh03`9_e~3JEftc@8CZ7%xMHTd3*cCH=bxj)3-NUKXMM z8OE*QbVra`TlVd3U2P`C+LSgAm%Mo&x-BVpU9O}d9(Qv^u6fq%nu90iDixVIC@`ns z#(kPn^GXs2wWUc;YvBRYj!)yb3r-xHj+ENEYzMlvd%N8-wHLdYs@QdcoZ@H=oe&S2 zt>SrjikegnXE2ZL`DvAoRgaQQv(w^nK3XOAGPxHe*I*C8C63QB2eaJPJ?mR$>@;?m zjcxYAWv8$Nr_zkNI^OIWRp1_M@(SvMH9T$Pm`AfUa#f}-!CFj5xjHD$H#m8to7CC1 z00*HOH*Y_yha4md6K4uqr!PRPYMHeaCOMgDR&+jVkZVOQ!o2xJ$j=Q&B3=f^vpYRV zRr;tCWZD?HS)9@4q-`?R9{0T6VUmWKmWlO|Dm9DQ>j)-l0RqNe*I!T0d5M$dyzE+L z0>>Gv?=&uYFRdDWPbi#N6H7TvUUmqny1``8!t8LUCK^LQjVsCyC$^$&9Lg9ts=4Vo zH!+!W;l8vP8?KszQZ7{tM>ZcY*i1MkPBUA)sO`BU>7M1!Pn*8MpC5ClZ^(T#w7d)l zpipIX*uXn+OvKI_F=PtFo}?0X7pEndA@;&gYfL`|WOf>i z9XF;^@&)9OS#W-Q3Yc$$wJ1(Y9>W=Pm-__A`#=!GH~zgZ$*Ph&V`af%vEs zhv&4eTyFj@+W}{4e+vfOX?vPib)GXG11`_vV&#=jG;t4~5s}Q(c4BOK<&_wux{(p+ zF&Wushb9A@Qs=b4I50XNZQ^1OY#w9$;>a_}Ip%rx0<-TL5sqDX6TAE3i>L4YGz+hI?SN;Y`{`rC(RlYuQ6C%34N87CBGGtwtr}@s zU<<&ju(1)6U?i?j4!0j`I`?!_DI(DEdETL>pmoO*nAE!bRb$c#SmAA6XTg>5}_N_P}e)25b{JL$unO-S<&oL#<{gIURP6Foj6$9le-$7kBI z8n(Csx{bN>z~?*8H|Ez~%2Z^3WTU#T1ThnEH$s=|Y^V!^T8v_&ci zsK7YL2tEZ=TyJJ%BpV0f!6;O5y{G>BiW+6^v2=zOfA(_sLQNgJ85l>l?tTsffMd1Joqgi4In8!k?yjod@_kc;k(9K@=1|NXLvE>g zRQK-w1T1FR7>clbNWI(GXTBhCSNX6ehawnc7Hk*$E^1k~8t&Y7XfH!x_h8x-+HaZf zsA1mi*f4V=>ZXO!`N4Q~s20=W-5;Af7Pi1_=DTgxvphZ6g=2uZ<;J#~s^>=-xxgIs zo7OZp&GhE6z$kbduoz!p=JR=PY76GJ9o*cyd^4u0`yXJfxx3BEMQ*K#UBT>;n>Eh8 zw$&%SugyzLgHQu=o~dSB#ynviadC`LgbR$!9xtkkEj&IQUAmX9xu;gGZoiq#c;{VA zNs)U<9b;kpPR7BHj-@BSq4*oD4OWL^;R#rph}q$j%^Hou!}snA64T(1CZTB~vv6a& zwQW3l;_!+S`wz7pMOr$Dy9UkmW+9uKaB`yGEz^(GQ6`YU;rO1wFt=ZHmkJqr0vO%y zTqd*ZSYj?)cP=_{c)4CiWcKjlsX3wyy6I^*h!nWFF1Nt_UXA1x#Bd5XmwoZ_w8L6e z2uCz8lAw+EsemtQT>9b|g#~xNr{IiUQ}ao04s>qpa8C(svv!&IssW^DGR+%L|Dpy{ zpXDCAhiAp%{K0UQnQjwbhh^cj8iEOWW-W9}lX-Dc#A0Gcm8s@&6qe!F7R>`q{wG;W)SnbZ_ z{i&a07<$xYa%#gd+|^?cW$xX*2|WQZO71o{7SA7U z+5J2$IeW;qUwkq=_{i{Zq-tckPZu9FaYJ(G^v5JR7u{eI4TW5X0 z7QYy_*RCxb+Yqj)stDCZE2|x=;-iVD53dT=2ge5Ep$c;YU7Qz+79nX|J7@@h2P~`# z#Vcdsx_Z3(L(KP>tGpHgw?&{V6t52qb>9-m92cAz3ZPLz@wEZ-?tuZ=83;WRti_#w zpspcS7d5ApXryNHfMMafcEdXrAS#V-2#ycMeU6D25&T@spCe!MA1jQ7Dv@<6;^8Tw Hb-n)&VkY@P literal 0 HcmV?d00001 From e9b0496045c564eab87a4ac7964b87462796bc44 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 8 Oct 2021 21:12:46 +0200 Subject: [PATCH 07/14] remove 'ingest.geoip.downloader.enabled' setting, since the default in test clusters is false. --- docs/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/build.gradle b/docs/build.gradle index 9540b840e7943..0a090d8124767 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -63,7 +63,6 @@ testClusters.matching { it.name == "integTest"}.configureEach { if (singleNode().testDistribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' setting 'indices.lifecycle.history_index_enabled', 'false' - setting 'ingest.geoip.downloader.enabled', 'false' keystorePassword 'keystore-password' } From 15b368d48f166d278de0670fbb5a25ee6833627c Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 8 Oct 2021 21:16:32 +0200 Subject: [PATCH 08/14] fix precommit --- docs/build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/build.gradle b/docs/build.gradle index 0a090d8124767..b4e9b052c50d0 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -115,6 +115,10 @@ tasks.named("integTest").configure { } } +tasks.named("forbiddenPatterns").configure { + exclude '**/*.mmdb' +} + tasks.named("buildRestTests").configure { docs = docsFileTree } From aabb43601870c591e901082e26f6c26a0a1a957b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Sun, 10 Oct 2021 23:53:15 +0200 Subject: [PATCH 09/14] added tests and fixed reloading --- .../ingest/geoip/GeoIpDownloaderIT.java | 33 +++++++ .../ingest/geoip/DatabaseRegistry.java | 24 ++++-- .../ingest/geoip/GeoIpProcessor.java | 4 + .../ingest/geoip/DatabaseRegistryTests.java | 41 ++++++++- .../geoip/GeoIpProcessorFactoryTests.java | 42 +++++++++ .../elasticsearch/ingest/IngestService.java | 11 ++- .../ingest/IngestServiceTests.java | 85 ++++++++++++++++++- 7 files changed, 223 insertions(+), 17 deletions(-) diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index b800955f546c9..c1bbd1cf4c3fc 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -50,6 +50,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -67,6 +68,7 @@ import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class GeoIpDownloaderIT extends AbstractGeoIpIT { @@ -307,6 +309,37 @@ public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception { }); } + public void testStartWithNoDatabases() throws Exception { + assumeTrue("only test with fixture to have stable results", ENDPOINT != null); + putPipeline(); + + // Behaviour without any databases loaded: + { + SimulateDocumentBaseResult result = simulatePipeline(); + assertThat(result.getFailure(), nullValue()); + assertThat(result.getIngestDocument(), notNullValue()); + Map source = result.getIngestDocument().getSourceAndMetadata(); + assertThat((Collection) source.get("tags"), containsInAnyOrder("_geoip_database_unavailable_GeoLite2-City.mmdb", + "_geoip_database_unavailable_GeoLite2-Country.mmdb", "_geoip_database_unavailable_GeoLite2-ASN.mmdb")); + } + + // Enable downloader: + Settings.Builder settings = Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), true); + assertAcked(client().admin().cluster().prepareUpdateSettings().setPersistentSettings(settings)); + { + assertBusy(() -> { + SimulateDocumentBaseResult result = simulatePipeline(); + assertThat(result.getFailure(), nullValue()); + assertThat(result.getIngestDocument(), notNullValue()); + Map source = result.getIngestDocument().getSourceAndMetadata(); + assertThat(source.get("tags"), nullValue()); + assertThat(((Map) source.get("ip-city")).get("city_name"), equalTo("Linköping")); + assertThat(((Map) source.get("ip-asn")).get("organization_name"), equalTo("Bredband2 AB")); + assertThat(((Map) source.get("ip-country")).get("country_name"), equalTo("Sweden")); + }); + } + } + private void verifyUpdatedDatabase() throws Exception { assertBusy(() -> { SimulateDocumentBaseResult result = simulatePipeline(); diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java index a765798404b05..6439dc008aa8f 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseRegistry.java @@ -53,8 +53,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.GZIPInputStream; @@ -91,7 +91,6 @@ public final class DatabaseRegistry implements Closeable { private IngestService ingestService; private final ConcurrentMap databases = new ConcurrentHashMap<>(); - private final AtomicBoolean initialized = new AtomicBoolean(false); DatabaseRegistry(Environment environment, Client client, GeoIpCache cache, Consumer genericExecutor) { this( @@ -317,17 +316,24 @@ void updateDatabase(String databaseFileName, String recordedMd5, Path file) { DatabaseReaderLazyLoader existing = databases.put(databaseFileName, loader); if (existing != null) { existing.close(); - } - LOGGER.info("successfully reloaded changed geoip database file [{}]", file); - if (databases.size() == 3 && initialized.compareAndSet(false, true)) { - List ids = ingestService.getPipelineWithProcessorType(GeoIpProcessor.DatabaseUnavailableProcessor.class); + } else { + // Loaded a database for the first time, so reload pipelines for which a database was not available: + Predicate predicate = p -> databaseFileName.equals(p.getDatabaseName()); + var ids = ingestService.getPipelineWithProcessorType(GeoIpProcessor.DatabaseUnavailableProcessor.class, predicate); if (ids.isEmpty() == false) { - for (String id : ids) { - ingestService.reloadPipeline(id); + for (var id : ids) { + try { + ingestService.reloadPipeline(id); + LOGGER.debug("successfully reloaded pipeline [{}] after downloading of database [{}] for the first time", + id, databaseFileName); + } catch (Exception e) { + LOGGER.debug((Supplier) () -> new ParameterizedMessage( + "failed to reload pipeline [{}] after downloading of database [{}]", id, databaseFileName), e); + } } - LOGGER.info("reloaded {} pipelines after successful initial downloading of databases", ids.size()); } } + LOGGER.info("successfully reloaded changed geoip database file [{}]", file); } catch (Exception e) { LOGGER.error((Supplier) () -> new ParameterizedMessage("failed to update database [{}]", databaseFileName), e); } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index c95a441a95172..d1d9cd788d2da 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -551,5 +551,9 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { public String getType() { return TYPE; } + + public String getDatabaseName() { + return databaseName; + } } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java index 602f7dde45aab..908ef873331c7 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseRegistryTests.java @@ -71,6 +71,7 @@ import java.util.UUID; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.IntStream; import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -80,13 +81,18 @@ import static org.elasticsearch.persistent.PersistentTasksCustomMetadata.TYPE; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; @LuceneTestCase.SuppressFileSystems(value = "ExtrasFS") // Don't randomly add 'extra' files to directory. @@ -97,6 +103,7 @@ public class DatabaseRegistryTests extends ESTestCase { private ThreadPool threadPool; private DatabaseRegistry databaseRegistry; private ResourceWatcherService resourceWatcherService; + private IngestService ingestService; @Before public void setup() throws IOException { @@ -111,9 +118,10 @@ public void setup() throws IOException { resourceWatcherService = new ResourceWatcherService(settings, threadPool); client = mock(Client.class); + ingestService = mock(IngestService.class); geoIpTmpDir = createTempDir(); databaseRegistry = new DatabaseRegistry(geoIpTmpDir, client, cache, localDatabases, Runnable::run); - databaseRegistry.initialize("nodeId", resourceWatcherService, mock(IngestService.class)); + databaseRegistry.initialize("nodeId", resourceWatcherService, ingestService); } @After @@ -138,11 +146,17 @@ public void testCheckDatabases() throws Exception { .routingTable(createIndexRoutingTable()) .build(); + int numPipelinesToBeReloaded = randomInt(4); + List pipelineIds = IntStream.range(0, numPipelinesToBeReloaded).mapToObj(String::valueOf).collect(Collectors.toList()); + when(ingestService.getPipelineWithProcessorType(any(), any())).thenReturn(pipelineIds); + assertThat(databaseRegistry.getDatabase("GeoIP2-City.mmdb"), nullValue()); + // Nothing should be downloaded, since the database is no longer valid (older than 30 days) databaseRegistry.checkDatabases(state); DatabaseReaderLazyLoader database = databaseRegistry.getDatabase("GeoIP2-City.mmdb"); assertThat(database, nullValue()); verify(client, times(0)).search(any()); + verify(ingestService, times(0)).reloadPipeline(anyString()); try (Stream files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) { assertEquals(0, files.count()); } @@ -158,10 +172,16 @@ public void testCheckDatabases() throws Exception { .localNodeId("_id1")) .routingTable(createIndexRoutingTable()) .build(); + // Database should be downloaded databaseRegistry.checkDatabases(state); database = databaseRegistry.getDatabase("GeoIP2-City.mmdb"); assertThat(database, notNullValue()); verify(client, times(10)).search(any()); + try (Stream files = Files.list(geoIpTmpDir.resolve("geoip-databases").resolve("nodeId"))) { + assertThat(files.count(), greaterThanOrEqualTo(1L)); + } + // First time GeoIP2-City.mmdb is downloaded, so a pipeline reload can happen: + verify(ingestService, times(numPipelinesToBeReloaded)).reloadPipeline(anyString()); //30 days check passed but we mocked mmdb data so parsing will fail expectThrows(InvalidDatabaseException.class, database::get); } @@ -272,6 +292,25 @@ public void testRetrieveDatabaseCorruption() throws Exception { verify(client, times(10)).search(any()); } + public void testUpdateDatabase() throws Exception { + int numPipelinesToBeReloaded = randomInt(4); + List pipelineIds = IntStream.range(0, numPipelinesToBeReloaded).mapToObj(String::valueOf).collect(Collectors.toList()); + when(ingestService.getPipelineWithProcessorType(any(), any())).thenReturn(pipelineIds); + + databaseRegistry.updateDatabase("_name", "_md5", geoIpTmpDir.resolve("some-file")); + + // Updating the first time may trigger a reload. + verify(ingestService, times(1)).addIngestClusterStateListener(any()); + verify(ingestService, times(1)).getPipelineWithProcessorType(any(), any()); + verify(ingestService, times(numPipelinesToBeReloaded)).reloadPipeline(anyString()); + verifyNoMoreInteractions(ingestService); + reset(ingestService); + + // Subsequent updates shouldn't trigger a reload. + databaseRegistry.updateDatabase("_name", "_md5", geoIpTmpDir.resolve("some-file")); + verifyZeroInteractions(ingestService); + } + private String mockSearches(String databaseName, int firstChunk, int lastChunk) throws IOException { String dummyContent = "test: " + databaseName; List data = gzip(databaseName, dummyContent, lastChunk - firstChunk + 1); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index bc14e58886561..f065cc0473864 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -48,6 +48,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -72,6 +73,7 @@ public void loadDatabaseReaders() throws IOException { copyDatabaseFiles(geoIpConfigDir, localDatabases); geoipTmpDir = createTempDir(); databaseRegistry = new DatabaseRegistry(geoipTmpDir, client, cache, localDatabases, Runnable::run); + databaseRegistry.initialize("nodeId", mock(ResourceWatcherService.class), mock(IngestService.class)); clusterService = mock(ClusterService.class); when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE); } @@ -423,6 +425,46 @@ public void testUpdateDatabaseWhileIngesting() throws Exception { } } + public void testDatabaseNotReadyYet() throws Exception { + GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseRegistry, clusterService); + + { + Map config = new HashMap<>(); + config.put("field", "source_field"); + config.put("database_file", "GeoLite2-City-Test.mmdb"); + + Map document = new HashMap<>(); + document.put("source_field", "89.160.20.128"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + GeoIpProcessor.DatabaseUnavailableProcessor processor = + (GeoIpProcessor.DatabaseUnavailableProcessor) factory.create(null, null, null, config); + processor.execute(ingestDocument); + assertThat(ingestDocument.getSourceAndMetadata().get("geoip"), nullValue()); + assertThat(ingestDocument.getSourceAndMetadata().get("tags"), + equalTo(List.of("_geoip_database_unavailable_GeoLite2-City-Test.mmdb"))); + } + + copyDatabaseFile(geoipTmpDir, "GeoLite2-City-Test.mmdb"); + databaseRegistry.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoipTmpDir.resolve("GeoLite2-City-Test.mmdb")); + + { + Map config = new HashMap<>(); + config.put("field", "source_field"); + config.put("database_file", "GeoLite2-City-Test.mmdb"); + + Map document = new HashMap<>(); + document.put("source_field", "89.160.20.128"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config); + processor.execute(ingestDocument); + assertThat(ingestDocument.getSourceAndMetadata().get("tags"), nullValue()); + Map geoData = (Map) ingestDocument.getSourceAndMetadata().get("geoip"); + assertThat(geoData.get("city_name"), equalTo("Linköping")); + } + } + private static void copyDatabaseFile(final Path path, final String databaseFilename) throws IOException { Files.copy( new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/" + databaseFilename)), diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 2b169d15075ee..a3f5738a19c7c 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -53,6 +53,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -67,6 +68,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.IntConsumer; +import java.util.function.Predicate; /** * Holder class for several ingest related services. @@ -869,12 +871,12 @@ public

List

getProcessorsInPipeline(String pipelineId, return processors; } - public

List getPipelineWithProcessorType(Class

clazz) { + public

Collection getPipelineWithProcessorType(Class

clazz, Predicate

predicate) { List matchedPipelines = new LinkedList<>(); for (PipelineHolder holder : pipelines.values()) { String pipelineId = holder.pipeline.getId(); List

processors = getProcessorsInPipeline(pipelineId, clazz); - if (processors.isEmpty() == false) { + if (processors.isEmpty() == false && processors.stream().anyMatch(predicate)) { matchedPipelines.add(pipelineId); } } @@ -887,8 +889,9 @@ public void reloadPipeline(String id) throws Exception { Pipeline updatedPipeline = Pipeline.create(id, holder.configuration.getConfigAsMap(), processorFactories, scriptService); synchronized (this) { - Map existingPipelines = this.pipelines; - existingPipelines.put(id, new PipelineHolder(holder.configuration, updatedPipeline)); + Map updatedPipelines = new HashMap<>(this.pipelines); + updatedPipelines.put(id, new PipelineHolder(holder.configuration, updatedPipeline)); + this.pipelines = Map.copyOf(updatedPipelines); } } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 087f961b61ac0..57f567b652717 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -42,10 +42,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xcontent.cbor.CborXContent; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; @@ -59,6 +56,9 @@ import org.elasticsearch.test.MockLogAppender; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool.Names; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.cbor.CborXContent; import org.hamcrest.CustomTypeSafeMatcher; import org.junit.Before; import org.mockito.ArgumentMatcher; @@ -86,6 +86,9 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.elasticsearch.core.Tuple.tuple; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; @@ -325,6 +328,82 @@ public void testGetProcessorsInPipeline() throws Exception { assertThat("pipeline with id [fakeID] does not exist", equalTo(e.getMessage())); } + public void testGetPipelineWithProcessorType() throws Exception { + IngestService ingestService = createWithProcessors(); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); + ClusterState previousClusterState = clusterState; + + PutPipelineRequest putRequest1 = new PutPipelineRequest("_id1", new BytesArray( + "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\", \"tag\": \"tag1\"}}," + + "{\"remove\" : {\"field\": \"_field\", \"tag\": \"tag2\"}}]}"), + XContentType.JSON); + clusterState = IngestService.innerPut(putRequest1, clusterState); + PutPipelineRequest putRequest2 = new PutPipelineRequest("_id2", new BytesArray( + "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\", \"tag\": \"tag2\"}}]}"), + XContentType.JSON); + clusterState = IngestService.innerPut(putRequest2, clusterState); + ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState)); + + assertThat(ingestService.getPipelineWithProcessorType(FakeProcessor.class, processor -> true), containsInAnyOrder("_id1", "_id2")); + assertThat(ingestService.getPipelineWithProcessorType(FakeProcessor.class, processor -> false), emptyIterable()); + assertThat(ingestService.getPipelineWithProcessorType(WrappingProcessorImpl.class, processor -> true), containsInAnyOrder("_id1")); + } + + public void testReloadPipeline() throws Exception { + boolean[] externalProperty = new boolean[] {false}; + + Map processorFactories = new HashMap<>(); + processorFactories.put("set", (factories, tag, description, config) -> { + String field = (String) config.remove("field"); + String value = (String) config.remove("value"); + if (externalProperty[0]) { + return new FakeProcessor("set", tag, description, (ingestDocument) ->ingestDocument.setFieldValue(field, value)); + } else { + return new AbstractProcessor(tag, description) { + @Override + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + throw new RuntimeException("reload me"); + } + + @Override + public String getType() { + return "set"; + } + }; + } + }); + + IngestService ingestService = createWithProcessors(processorFactories); + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); + ClusterState previousClusterState = clusterState; + + PutPipelineRequest putRequest1 = new PutPipelineRequest("_id1", new BytesArray( + "{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\", \"tag\": \"tag1\"}}]}"), + XContentType.JSON); + clusterState = IngestService.innerPut(putRequest1, clusterState); + ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState)); + + { + Exception[] exceptionHolder = new Exception[1]; + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + ingestService.getPipeline("_id1").execute(ingestDocument, (ingestDocument1, e) -> exceptionHolder[0] = e); + assertThat(exceptionHolder[0], notNullValue()); + assertThat(exceptionHolder[0].getMessage(), containsString("reload me")); + assertThat(ingestDocument.getSourceAndMetadata().get("_field"), nullValue()); + } + + externalProperty[0] = true; + ingestService.reloadPipeline("_id1"); + + { + Exception[] holder = new Exception[1]; + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + ingestService.getPipeline("_id1").execute(ingestDocument, (ingestDocument1, e) -> holder[0] = e); + assertThat(holder[0], nullValue()); + assertThat(ingestDocument.getSourceAndMetadata().get("_field"), equalTo("_value")); + } + } + public void testGetProcessorsInPipelineComplexConditional() throws Exception { LongSupplier relativeTimeProvider = mock(LongSupplier.class); String scriptName = "conditionalScript"; From 5582ab36299a777e035623a3d9890d1a3fe5a825 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 11 Oct 2021 09:49:51 +0200 Subject: [PATCH 10/14] simplify --- .../java/org/elasticsearch/ingest/IngestService.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index a3f5738a19c7c..0c27512128ea3 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -883,16 +883,13 @@ public

Collection getPipelineWithProcessorType(Cla return matchedPipelines; } - public void reloadPipeline(String id) throws Exception { + public synchronized void reloadPipeline(String id) throws Exception { PipelineHolder holder = pipelines.get(id); - Pipeline updatedPipeline = Pipeline.create(id, holder.configuration.getConfigAsMap(), processorFactories, scriptService); - synchronized (this) { - Map updatedPipelines = new HashMap<>(this.pipelines); - updatedPipelines.put(id, new PipelineHolder(holder.configuration, updatedPipeline)); - this.pipelines = Map.copyOf(updatedPipelines); - } + Map updatedPipelines = new HashMap<>(this.pipelines); + updatedPipelines.put(id, new PipelineHolder(holder.configuration, updatedPipeline)); + this.pipelines = Map.copyOf(updatedPipelines); } private static Pipeline substitutePipeline(String id, ElasticsearchParseException e) { From c393b3e9f6b63b72d330c8a401993ead206b81a5 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 11 Oct 2021 16:01:25 +0200 Subject: [PATCH 11/14] altered assertions --- .../ingest/geoip/GeoIpDownloaderIT.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java index c1bbd1cf4c3fc..1d02a5b7f22fa 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java @@ -64,8 +64,10 @@ import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -319,8 +321,8 @@ public void testStartWithNoDatabases() throws Exception { assertThat(result.getFailure(), nullValue()); assertThat(result.getIngestDocument(), notNullValue()); Map source = result.getIngestDocument().getSourceAndMetadata(); - assertThat((Collection) source.get("tags"), containsInAnyOrder("_geoip_database_unavailable_GeoLite2-City.mmdb", - "_geoip_database_unavailable_GeoLite2-Country.mmdb", "_geoip_database_unavailable_GeoLite2-ASN.mmdb")); + assertThat(source, hasEntry("tags", List.of("_geoip_database_unavailable_GeoLite2-City.mmdb", + "_geoip_database_unavailable_GeoLite2-Country.mmdb", "_geoip_database_unavailable_GeoLite2-ASN.mmdb"))); } // Enable downloader: @@ -331,8 +333,12 @@ public void testStartWithNoDatabases() throws Exception { SimulateDocumentBaseResult result = simulatePipeline(); assertThat(result.getFailure(), nullValue()); assertThat(result.getIngestDocument(), notNullValue()); - Map source = result.getIngestDocument().getSourceAndMetadata(); - assertThat(source.get("tags"), nullValue()); + Map source = result.getIngestDocument().getSourceAndMetadata(); + assertThat(source, not(hasKey("tags"))); + assertThat(source, hasKey("ip-city")); + assertThat(source, hasKey("ip-asn")); + assertThat(source, hasKey("ip-country")); + assertThat(((Map) source.get("ip-city")).get("city_name"), equalTo("Linköping")); assertThat(((Map) source.get("ip-asn")).get("organization_name"), equalTo("Bredband2 AB")); assertThat(((Map) source.get("ip-country")).get("country_name"), equalTo("Sweden")); From 1ab5eed92f84fad73f54727c0c9c2e96b598774a Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 13 Oct 2021 11:04:55 +0200 Subject: [PATCH 12/14] updated jdoc --- .../java/org/elasticsearch/ingest/geoip/LocalDatabases.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java index 4f15b40c32557..60958cc8a65af 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/LocalDatabases.java @@ -30,9 +30,8 @@ import java.util.stream.Stream; /** - * Keeps track of the databases locally available to a node: - * 1) User provided databases from the ES_HOME/config/ingest-geoip directory. This directory is monitored - * and files updates are picked up and may cause databases being loaded or removed at runtime. + * Keeps track of user provided databases in the ES_HOME/config/ingest-geoip directory. + * This directory is monitored and files updates are picked up and may cause databases being loaded or removed at runtime. */ final class LocalDatabases implements Closeable { From 5d8964410dd08f42d1de5d41e3d81b9ee8fd6566 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 13 Oct 2021 13:48:53 +0200 Subject: [PATCH 13/14] Add note to the ingest section of the migration docs. --- .../migration/migrate_8_0/ingest.asciidoc | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/docs/reference/migration/migrate_8_0/ingest.asciidoc b/docs/reference/migration/migrate_8_0/ingest.asciidoc index d827a15de92ed..5437b3b7d7c93 100644 --- a/docs/reference/migration/migrate_8_0/ingest.asciidoc +++ b/docs/reference/migration/migrate_8_0/ingest.asciidoc @@ -18,4 +18,31 @@ Common Schema (ECS)] fields, regardless of the `ecs` value. To avoid deprecation warnings, remove the parameter from your ingest pipelines. If a pipeline specifies an `ecs` value, the value is ignored. ==== + +.The default Maxmind geoip databases have been removed. +[%collapsible] +==== +*Details* + +The default Maxmind geoip databases that shipped by default with Elasticsearch +have been removed. These databases are out dated and stale and using these +databases will likely result in incorrect geoip lookups. + +By default since 7.13, these pre-packaged geoip databases +were used in case no database were specified in the config directory or before +the geoip downloader downloaded the geoip databases. If the geoip database +downloader completed downloading the new databases then these pre-packaged +databases are no longer used. + +*Impact* + +If the geoip downloader is disabled and no geoip databases are provided +in the config directory of each ingest node then the geoip processor will +no longer perform geoip lookups and tag these documents that the requested +database is no longer available. + +After a cluster has been started and before the geoip downloader has completed +downloading the most up to data databases, the geoip processor will not perform +any geoip lookups and tag documents that the requested database is not available. +After the geoip downloader has completed downloading the most up to data databases +then the geoip processor will function as normal. +==== //end::notable-breaking-changes[] From bbf8cb80b1361d42d434e513a2bf3aec3d26e06f Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 13 Oct 2021 13:51:46 +0200 Subject: [PATCH 14/14] iter --- docs/reference/migration/migrate_8_0/ingest.asciidoc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/reference/migration/migrate_8_0/ingest.asciidoc b/docs/reference/migration/migrate_8_0/ingest.asciidoc index 5437b3b7d7c93..b9308988b870d 100644 --- a/docs/reference/migration/migrate_8_0/ingest.asciidoc +++ b/docs/reference/migration/migrate_8_0/ingest.asciidoc @@ -29,20 +29,21 @@ databases will likely result in incorrect geoip lookups. By default since 7.13, these pre-packaged geoip databases were used in case no database were specified in the config directory or before -the geoip downloader downloaded the geoip databases. If the geoip database +the geoip downloader downloaded the geoip databases. When the geoip database downloader completed downloading the new databases then these pre-packaged -databases are no longer used. +databases were no longer used. *Impact* + If the geoip downloader is disabled and no geoip databases are provided in the config directory of each ingest node then the geoip processor will -no longer perform geoip lookups and tag these documents that the requested -database is no longer available. +no longer perform geoip lookups and tag these documents with the fact that +the requested database is no longer available. After a cluster has been started and before the geoip downloader has completed downloading the most up to data databases, the geoip processor will not perform any geoip lookups and tag documents that the requested database is not available. After the geoip downloader has completed downloading the most up to data databases -then the geoip processor will function as normal. +then the geoip processor will function as normal. The window of time that the +geoip processor can't do geoip lookups after cluster startup should be very small. ==== //end::notable-breaking-changes[]