[7.x] Option to disable device type parsing in user agent processor (#…

…71625) (#71970)

docs/reference/ingest/common-log-format-example.asciidoc

@@ -247,8 +247,7 @@ The API returns:
             },
             "name": "Chrome",
             "device": {
-              "name": "Mac",
-              "type": "Desktop"
+              "name": "Mac"
             },
             "version": "52.0.2743.116"
           }

docs/reference/ingest/processors/user-agent.asciidoc

@@ -71,9 +71,8 @@ Which returns
         "full": "Mac OS X 10.10.5"
       },
       "device" : {
-        "name" : "Mac",
-        "type" : "Desktop"
-      },
+        "name" : "Mac"
+      }
     }
   }
 }

modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java

@@ -153,14 +153,14 @@ String getName() {
         return name;
     }
 
-    public Details parse(String agentString) {
+    public Details parse(String agentString, boolean extractDeviceType) {
         Details details = cache.get(name, agentString);
 
         if (details == null) {
             VersionedName userAgent = findMatch(uaPatterns, agentString);
             VersionedName operatingSystem = findMatch(osPatterns, agentString);
             VersionedName device = findMatch(devicePatterns, agentString);
-            String deviceType = deviceTypeParser.findDeviceType(agentString, userAgent, operatingSystem, device);
+            String deviceType = extractDeviceType ? deviceTypeParser.findDeviceType(agentString, userAgent, operatingSystem, device) : null;
             details = new Details(userAgent, operatingSystem, device, deviceType);
             cache.put(name, agentString, details);
         }

modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java

@@ -41,20 +41,26 @@ public class UserAgentProcessor extends AbstractProcessor {
     private final String targetField;
     private final Set<Property> properties;
     private final UserAgentParser parser;
+    private final boolean extractDeviceType;
     private final boolean ignoreMissing;
     private final boolean useECS;
 
     public UserAgentProcessor(String tag, String description, String field, String targetField, UserAgentParser parser,
-                              Set<Property> properties, boolean ignoreMissing, boolean useECS) {
+                              Set<Property> properties, boolean extractDeviceType, boolean ignoreMissing, boolean useECS) {
         super(tag, description);
         this.field = field;
         this.targetField = targetField;
         this.parser = parser;
         this.properties = properties;
+        this.extractDeviceType = extractDeviceType;
         this.ignoreMissing = ignoreMissing;
         this.useECS = useECS;
     }
 
+    boolean isExtractDeviceType() {
+        return extractDeviceType;
+    }
+
     boolean isIgnoreMissing() {
         return ignoreMissing;
     }
@@ -69,7 +75,7 @@ public IngestDocument execute(IngestDocument ingestDocument) {
             throw new IllegalArgumentException("field [" + field + "] is null, cannot parse user-agent.");
         }
 
-        Details uaClient = parser.parse(userAgent);
+        Details uaClient = parser.parse(userAgent, extractDeviceType);
 
         Map<String, Object> uaDetails = new HashMap<>();
 
@@ -126,22 +132,26 @@ public IngestDocument execute(IngestDocument ingestDocument) {
                                 uaDetails.put("os", osDetails);
                             }
                         }
-                        break;
-                    case DEVICE:
-                        Map<String, String> deviceDetails = new HashMap<>(1);
-                        if (uaClient.device != null && uaClient.device.name != null) {
-                            deviceDetails.put("name", uaClient.device.name);
+                    break;
+                case DEVICE:
+                    Map<String, String> deviceDetails = new HashMap<>(1);
+                    if (uaClient.device != null && uaClient.device.name != null) {
+                        deviceDetails.put("name", uaClient.device.name);
+                        if (extractDeviceType) {
                             deviceDetails.put("type", uaClient.deviceType);
-                        } else {
-                            deviceDetails.put("name", "Other");
+                        }
+                    } else {
+                        deviceDetails.put("name", "Other");
+                        if (extractDeviceType) {
                             if (uaClient.deviceType != null) {
                                 deviceDetails.put("type", uaClient.deviceType);
                             } else {
                                 deviceDetails.put("type", "Other");
                             }
                         }
-                        uaDetails.put("device", deviceDetails);
-                        break;
+                    }
+                    uaDetails.put("device", deviceDetails);
+                    break;
                 }
             }
         } else {
@@ -204,13 +214,17 @@ public IngestDocument execute(IngestDocument ingestDocument) {
                         Map<String, String> deviceDetails = new HashMap<>(1);
                         if (uaClient.device != null && uaClient.device.name != null) {
                             deviceDetails.put("name", uaClient.device.name);
-                            deviceDetails.put("type", uaClient.deviceType);
+                            if (extractDeviceType) {
+                                deviceDetails.put("type", uaClient.deviceType);
+                            }
                         } else {
                             deviceDetails.put("name", "Other");
-                            if (uaClient.deviceType != null) {
-                                deviceDetails.put("type", uaClient.deviceType);
-                            } else {
-                                deviceDetails.put("type", "Other");
+                            if (extractDeviceType) {
+                                if (uaClient.deviceType != null) {
+                                    deviceDetails.put("type", uaClient.deviceType);
+                                } else {
+                                    deviceDetails.put("type", "Other");
+                                }
                             }
                         }
                         uaDetails.put("device", deviceDetails);
@@ -294,6 +308,7 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent");
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
+            boolean extractDeviceType = readBooleanProperty(TYPE, processorTag, config, "extract_device_type", false);
             boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
             boolean useECS = readBooleanProperty(TYPE, processorTag, config, "ecs", true);
 
@@ -323,7 +338,17 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
                     "format is deprecated and will be removed in 8.0, set to true or remove to use the non-deprecated format");
             }
 
-            return new UserAgentProcessor(processorTag, description, field, targetField, parser, properties, ignoreMissing, useECS);
+            return new UserAgentProcessor(
+                processorTag,
+                description,
+                field,
+                targetField,
+                parser,
+                properties,
+                extractDeviceType,
+                ignoreMissing,
+                useECS
+            );
         }
     }
 

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java

@@ -82,6 +82,7 @@ public void testBuildDefaults() throws Exception {
         assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0));
         assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class)));
+        assertFalse(processor.isExtractDeviceType());
         assertFalse(processor.isIgnoreMissing());
         assertTrue(processor.isUseECS());
     }
@@ -135,6 +136,19 @@ public void testBuildRegexFile() throws Exception {
         assertThat(processor.getUaParser().getDevicePatterns().size(), equalTo(0));
     }
 
+    public void testBuildExtractDeviceType() throws Exception {
+        UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
+        boolean extractDeviceType = randomBoolean();
+
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "_field");
+        config.put("extract_device_type", extractDeviceType);
+
+        UserAgentProcessor processor = factory.create(null, null, null, config);
+        assertThat(processor.getField(), equalTo("_field"));
+        assertThat(processor.isExtractDeviceType(), equalTo(extractDeviceType));
+    }
+
     public void testBuildNonExistingRegexFile() throws Exception {
         UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers);
 

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java

@@ -40,12 +40,12 @@ public static void setupProcessor() throws IOException {
         UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, deviceTypeRegexStream, new UserAgentCache(1000));
 
         processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", parser,
-                EnumSet.allOf(UserAgentProcessor.Property.class), false, true);
+                EnumSet.allOf(UserAgentProcessor.Property.class), true, false, true);
     }
 
     public void testNullValueWithIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), true, true);
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, true, true);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
             Collections.singletonMap("source_field", null));
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
@@ -55,7 +55,7 @@ public void testNullValueWithIgnoreMissing() throws Exception {
 
     public void testNonExistentWithIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), true, true);
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, true, true);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         processor.execute(ingestDocument);
@@ -64,7 +64,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception {
 
     public void testNullWithoutIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), false, true);
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, false, true);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(),
             Collections.singletonMap("source_field", null));
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
@@ -74,7 +74,7 @@ public void testNullWithoutIgnoreMissing() throws Exception {
 
     public void testNonExistentWithoutIgnoreMissing() throws Exception {
         UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null,
-            EnumSet.allOf(UserAgentProcessor.Property.class), false, true);
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, false, true);
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
@@ -244,4 +244,34 @@ public void testUnknown() throws Exception {
         device.put("type", "Other");
         assertThat(target.get("device"), is(device));
     }
+
+    @SuppressWarnings("unchecked")
+    public void testExtractDeviceTypeDisabled() {
+        Map<String, Object> document = new HashMap<>();
+        document.put("source_field",
+            "Something I made up v42.0.1");
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
+
+        InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yml");
+        InputStream deviceTypeRegexStream = UserAgentProcessor.class.getResourceAsStream("/device_type_regexes.yml");
+        UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, deviceTypeRegexStream, new UserAgentCache(1000));
+        UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", parser,
+            EnumSet.allOf(UserAgentProcessor.Property.class), false, false, true);
+        processor.execute(ingestDocument);
+        Map<String, Object> data = ingestDocument.getSourceAndMetadata();
+
+        assertThat(data, hasKey("target_field"));
+        Map<String, Object> target = (Map<String, Object>) data.get("target_field");
+
+        assertThat(target.get("name"), is("Other"));
+        assertNull(target.get("major"));
+        assertNull(target.get("minor"));
+        assertNull(target.get("patch"));
+        assertNull(target.get("build"));
+
+        assertNull(target.get("os"));
+        Map<String, String> device = new HashMap<>();
+        device.put("name", "Other");
+        assertThat(target.get("device"), is(device));
+    }
 }

modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml

@@ -32,7 +32,6 @@
   - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
   - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} }
   - match: { _source.user_agent.version: "33.0.1750.149" }
-  - match: { _source.user_agent.device: {"name": "Mac", type: "Desktop" }}
 
 ---
 "Test user agent processor with parameters":

modules/ingest-user-agent/src/yamlRestTest/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml

@@ -31,6 +31,5 @@
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
   - match: { _source.user_agent.name: "Test" }
-  - match: { _source.user_agent.device: {"name": "Other", "type": "Other" }}
   - is_false: _source.user_agent.os
   - is_false: _source.user_agent.version