Add option to not replace "." with "_" in field names

Elasticsearch does not allow "." characters in field names since version 2.0.
Support has been restored since version 5.0.
For compatibility, graylog replaces "." with "_".
However, when Elasticsearch >= 5.0 is used, this is unnecessary.
For instance, Wazuh Indexer from the Wazuh project is forked from
Opensearch 1.3. The character replacement causes issues with Wazuh,
as the Dashboard expects dots as separator in the field name.

This adds the option `replace_dots_in_field_names` to revert this behavior and
allow the use of ".". The replacement is enabled by default for
compatibility with existing graylog configurations.

The extractor configuration in the web interface has been modified to show
a warning when the user inputs a "." in the "Key separator" field.

Closes: Graylog2#4583
Closes: Graylog2#6588
Closes: Graylog2#13043
Closes: Graylog2#14901
Bug: elastic/elasticsearch#19443

graylog2-server/src/main/java/org/graylog2/configuration/ElasticsearchConfiguration.java

@@ -274,6 +274,9 @@ public Period getTimeSizeOptimizingRotationFixedLeeway() {
         return timeSizeOptimizingRotationFixedLeeway;
     }
 
+    @Parameter(value = "replace_dots_in_field_names")
+    private Boolean replaceDotsInFieldNames = true;
+
     public boolean isDisableVersionCheck() {
         return disableVersionCheck;
     }
@@ -333,4 +336,8 @@ MAX_INDEX_RETENTION_PERIOD, getMaxIndexRetentionPeriod())
             );
         }
     }
+
+    public Boolean getReplaceDotsInFieldNames() {
+        return replaceDotsInFieldNames;
+    }
 }

graylog2-server/src/main/java/org/graylog2/plugin/Message.java

@@ -34,10 +34,12 @@
 import org.apache.commons.lang3.StringUtils;
 import org.graylog.failure.FailureCause;
 import org.graylog.failure.ProcessingFailureCause;
+import org.graylog2.configuration.ElasticsearchConfiguration;
 import org.graylog2.indexer.IndexSet;
 import org.graylog2.indexer.messages.Indexable;
 import org.graylog2.plugin.streams.Stream;
 import org.graylog2.plugin.utilities.date.DateTimeConverter;
+import org.graylog2.shared.bindings.GuiceInjectorHolder;
 import org.graylog2.shared.utilities.ExceptionUtils;
 import org.joda.time.DateTime;
 import org.slf4j.Logger;
@@ -398,18 +400,20 @@ public DateTime getTimestamp() {
 
     @Override
     public Map<String, Object> toElasticSearchObject(ObjectMapper objectMapper, @Nonnull final Meter invalidTimestampMeter) {
+        final ElasticsearchConfiguration elasticsearchConfiguration =
+            GuiceInjectorHolder.getInjector().getInstance(ElasticsearchConfiguration.class);
         final Map<String, Object> obj = Maps.newHashMapWithExpectedSize(REQUIRED_FIELDS.size() + fields.size());
 
         for (Map.Entry<String, Object> entry : fields.entrySet()) {
             final String key = entry.getKey();
             if (key.equals(FIELD_ID)) {
                 continue;
             }
-
             final Object value = entry.getValue();
-            // Elasticsearch does not allow "." characters in keys since version 2.0.
+            // Elasticsearch does not allow "." characters in keys from versions 2.0 to 5.0 (excluded).
             // See: https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_mapping_changes.html#_field_names_may_not_contain_dots
-            if (key.contains(".")) {
+            // See: https://www.elastic.co/guide/en/elasticsearch/reference/5.0/release-notes-5.0.0.html#enhancement-5.0.0
+            if (key.contains(".") && elasticsearchConfiguration.getReplaceDotsInFieldNames()) {
                 final String newKey = key.replace('.', KEY_REPLACEMENT_CHAR);
 
                 // If the message already contains the transformed key, we skip the field and emit a warning.

graylog2-server/src/main/java/org/graylog2/shared/bindings/GuiceInjectorHolder.java

@@ -45,6 +45,10 @@ public static void resetInjector() {
         injector = null;
     }
 
+    public static void clearInjector() {
+        injector = null;
+    }
+
     public static Injector getInjector() {
         if (injector == null) {
             throw new IllegalStateException("No injector available. Please call createInjector() first.");

graylog2-server/src/test/java/org/graylog2/plugin/MessageTest.java

@@ -24,11 +24,15 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
+import com.google.inject.AbstractModule;
+import com.google.inject.Module;
 import org.graylog.failure.FailureCause;
 import org.graylog.failure.ProcessingFailureCause;
+import org.graylog2.configuration.ElasticsearchConfiguration;
 import org.graylog2.indexer.IndexSet;
 import org.graylog2.plugin.streams.Stream;
 import org.graylog2.shared.SuppressForbidden;
+import org.graylog2.shared.bindings.GuiceInjectorHolder;
 import org.graylog2.shared.bindings.providers.ObjectMapperProvider;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeUtils;
@@ -75,6 +79,7 @@ public class MessageTest {
     @Rule
     public final MockitoRule mockitoRule = MockitoJUnit.rule();
 
+    private final ElasticsearchConfiguration elasticsearchConfiguration = mock(ElasticsearchConfiguration.class);
     private final ObjectMapper objectMapper = new ObjectMapperProvider().get();
     private Message message;
     private DateTime originalTimestamp;
@@ -85,15 +90,28 @@ public class MessageTest {
     public void setUp() {
         DateTimeUtils.setCurrentMillisFixed(1524139200000L);
 
+        Module configModule = new AbstractModule() {
+            // Inject custom ElasticsearchConfiguration
+            @Override
+            protected void configure() {
+                bind(ElasticsearchConfiguration.class).toInstance(elasticsearchConfiguration);
+            }
+        };
+        GuiceInjectorHolder.clearInjector();
+        GuiceInjectorHolder.createInjector(Collections.singletonList(configModule));
+
         metricRegistry = new MetricRegistry();
         originalTimestamp = Tools.nowUTC();
         message = new Message("foo", "bar", originalTimestamp);
         invalidTimestampMeter = metricRegistry.meter("test");
 
+        when(elasticsearchConfiguration.getReplaceDotsInFieldNames())
+                .thenReturn(true); // Set to default value
     }
 
     @After
     public void tearDown() {
+        GuiceInjectorHolder.clearInjector();
         DateTimeUtils.setCurrentMillisSystem();
     }
 
@@ -430,6 +448,25 @@ public void testToElasticSearchObjectWithInvalidKey() throws Exception {
         assertThat(streams).isEmpty();
     }
 
+    @Test
+    public void testToElasticSearchObjectWithDotReplacementDisabled() throws Exception {
+        when(elasticsearchConfiguration.getReplaceDotsInFieldNames()).thenReturn(false);
+        message.addField("field.3", "dot");
+
+        final Map<String, Object> object = message.toElasticSearchObject(objectMapper, invalidTimestampMeter);
+
+        assertEquals("#toElasticsearchObject() should keep \".\" in keys when replace_dots_in_field_names = false",
+                "dot", object.get("field.3"));
+
+        assertEquals("foo", object.get("message"));
+        assertEquals("bar", object.get("source"));
+        assertEquals(Tools.buildElasticSearchTimeFormat((DateTime) message.getField("timestamp")), object.get("timestamp"));
+
+        @SuppressWarnings("unchecked")
+        final Collection<String> streams = (Collection<String>) object.get("streams");
+        assertThat(streams).isEmpty();
+    }
+
     @Test
     public void testToElasticSearchObjectWithoutDateTimeTimestamp() throws Exception {
         message.addField("timestamp", "time!");

graylog2-web-interface/src/components/extractors/extractors_configuration/JSONExtractorConfiguration.jsx

@@ -100,6 +100,12 @@ const JSONExtractorConfiguration = createReactClass({
     return this.state.trying || !this.props.exampleMessage;
   },
 
+  _keySeparatorError(separator) {
+    return separator.includes(".")
+      ? "Warning: Elasticsearch does not allow '.' in field names from version 2.0 up to 5.0 (excluded)"
+      : null
+  },
+
   render() {
     return (
       <div>
@@ -129,6 +135,7 @@ const JSONExtractorConfiguration = createReactClass({
                defaultValue={this.state.configuration.key_separator}
                required
                onChange={this._onChange('key_separator')}
+               error={this._keySeparatorError(this.state.configuration.key_separator)}
                help={<span>What string to use to concatenate different keys of a nested JSON object (only used if <em>not</em> flattened).</span>} />
 
         <Input type="text"

misc/graylog.conf

@@ -406,6 +406,13 @@ allow_leading_wildcard_searches = false
 # should only be enabled after making sure your Elasticsearch cluster has enough memory.
 allow_highlighting = false
 
+# Replace "." characters in keys with "_". Dots are not allowed by Elasticsearch version 2.0 up to 5.0 (excluded).
+# Support have been restored in Elasticsearch 5.0.
+# See: https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_mapping_changes.html#_field_names_may_not_contain_dots
+# See: https://www.elastic.co/guide/en/elasticsearch/reference/5.0/release-notes-5.0.0.html#enhancement-5.0.0
+# Default: true
+#replace_dots_in_field_names = true
+
 # Global timeout for index optimization (force merge) requests.
 # Default: 1h
 #elasticsearch_index_optimization_timeout = 1h