SQL: change the way unsupported data types fields are handled (elasti…

…c#50823)

The hierarchy of fields/sub-fields under a field that is of an unsupported data type will be marked as unsupported as well. Until this change, the behavior was to set the unsupported data type field's hierarchy as empty.

Example, considering the following hierarchy of fields/subfields  a -> b -> c -> d, if b would be of type "foo", then b, c and d will be marked as unsupported.

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java

@@ -358,11 +358,12 @@ private static EsField createField(String fieldName, Map<String, Map<String, Fie
 
         int dot = fieldName.lastIndexOf('.');
         String fullFieldName = fieldName;
+        EsField parent = null;
 
         if (dot >= 0) {
             String parentName = fieldName.substring(0, dot);
             fieldName = fieldName.substring(dot + 1);
-            EsField parent = flattedMapping.get(parentName);
+            parent = flattedMapping.get(parentName);
             if (parent == null) {
                 Map<String, FieldCapabilities> map = globalCaps.get(parentName);
                 Function<String, EsField> fieldFunction;
@@ -387,7 +388,22 @@ private static EsField createField(String fieldName, Map<String, Map<String, Fie
         }
 
         EsField esField = field.apply(fieldName);
-
+
+        if (parent != null && parent instanceof UnsupportedEsField) {
+            UnsupportedEsField unsupportedParent = (UnsupportedEsField) parent;
+            String inherited = unsupportedParent.getInherited();
+            String type = unsupportedParent.getOriginalType();
+
+            if (inherited == null) {
+                // mark the sub-field as unsupported, just like its parent, setting the first unsupported parent as the current one
+                esField = new UnsupportedEsField(esField.getName(), type, unsupportedParent.getName(), esField.getProperties());
+            } else {
+                // mark the sub-field as unsupported, just like its parent, but setting the first unsupported parent
+                // as the parent's first unsupported grandparent
+                esField = new UnsupportedEsField(esField.getName(), type, inherited, esField.getProperties());
+            }
+        }
+
         parentProps.put(fieldName, esField);
         flattedMapping.put(fullFieldName, esField);
 
@@ -408,7 +424,7 @@ private static EsField createField(String fieldName, String typeName, Map<String
             case DATETIME:
                 return new DateEsField(fieldName, props, isAggregateable);
             case UNSUPPORTED:
-                return new UnsupportedEsField(fieldName, typeName);
+                return new UnsupportedEsField(fieldName, typeName, null, props);
             default:
                 return new EsField(fieldName, esType, props, isAggregateable, isAlias);
         }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java

@@ -73,7 +73,7 @@ private static void walkMapping(String name, Object value, Map<String, EsField>
                     properties = Collections.emptyMap();
                 }
             } else {
-                properties = Collections.emptyMap();
+                properties = fromEs(content);
             }
             boolean docValues = boolSetting(content.get("doc_values"), esDataType.defaultDocValues);
             final EsField field;
@@ -91,7 +91,8 @@ private static void walkMapping(String name, Object value, Map<String, EsField>
                     break;
                 case UNSUPPORTED:
                     String type = content.get("type").toString();
-                    field = new UnsupportedEsField(name, type);
+                    field = new UnsupportedEsField(name, type, null, properties);
+                    propagateUnsupportedType(name, type, properties);
                     break;
                 default:
                     field = new EsField(name, esDataType, properties, docValues);
@@ -113,4 +114,21 @@ private static boolean boolSetting(Object value, boolean defaultValue) {
     private static int intSetting(Object value, int defaultValue) {
         return value == null ? defaultValue : Integer.parseInt(value.toString());
     }
+
+    private static void propagateUnsupportedType(String inherited, String originalType, Map<String, EsField> properties) {
+        if (properties != null && properties.isEmpty() == false) {
+            for (Entry<String, EsField> entry : properties.entrySet()) {
+                EsField field = entry.getValue();
+                UnsupportedEsField u;
+                if (field instanceof UnsupportedEsField) {
+                    u = (UnsupportedEsField) field;
+                    u = new UnsupportedEsField(u.getName(), originalType, inherited, u.getProperties());
+                } else {
+                    u = new UnsupportedEsField(field.getName(), originalType, inherited, field.getProperties());
+                }
+                entry.setValue(u);
+                propagateUnsupportedType(inherited, originalType, u.getProperties());
+            }
+        }
+    }
 }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/UnsupportedEsField.java

@@ -5,24 +5,40 @@
  */
 package org.elasticsearch.xpack.ql.type;
 
-import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 
 /**
- * SQL-related information about an index field that cannot be supported by SQL
+ * SQL-related information about an index field that cannot be supported by SQL.
+ * All the subfields (properties) of an unsupported type should also be unsupported.
  */
 public class UnsupportedEsField extends EsField {
 
-    private String originalType;
-
+    private final String originalType;
+    private final String inherited; // for fields belonging to parents (or grandparents) that have an unsupported type
+
     public UnsupportedEsField(String name, String originalType) {
-        super(name, DataType.UNSUPPORTED, Collections.emptyMap(), false);
+        this(name, originalType, null, new TreeMap<>());
+    }
+
+    public UnsupportedEsField(String name, String originalType, String inherited, Map<String, EsField> properties) {
+        super(name, DataType.UNSUPPORTED, properties, false);
         this.originalType = originalType;
+        this.inherited = inherited;
     }
 
     public String getOriginalType() {
         return originalType;
     }
+
+    public String getInherited() {
+        return inherited;
+    }
+
+    public boolean hasInherited() {
+        return inherited != null;
+    }
 
     @Override
     public boolean equals(Object o) {
@@ -36,11 +52,12 @@ public boolean equals(Object o) {
             return false;
         }
         UnsupportedEsField that = (UnsupportedEsField) o;
-        return Objects.equals(originalType, that.originalType);
+        return Objects.equals(originalType, that.originalType)
+                && Objects.equals(inherited, that.inherited);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), originalType);
+        return Objects.hash(super.hashCode(), originalType, inherited);
     }
 }

x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/type/TypesTests.java

@@ -188,6 +188,10 @@ public void testUnsupportedTypes() {
         Map<String, EsField> mapping = loadMapping("mapping-unsupported.json");
         EsField dt = mapping.get("range");
         assertThat(dt.getDataType().typeName(), is("unsupported"));
+        dt = mapping.get("time_frame");
+        assertThat(dt.getDataType().typeName(), is("unsupported"));
+        dt = mapping.get("flat");
+        assertThat(dt.getDataType().typeName(), is("unsupported"));
     }
 
     public static Map<String, EsField> loadMapping(String name) {

x-pack/plugin/ql/src/test/resources/mapping-multi-field-with-nested.json

@@ -7,6 +7,26 @@
         "unsupported" : { "type" : "ip_range" },
         "date" : { "type" : "date"},
         "shape": { "type" : "geo_shape" },
+        "x" : {
+            "type" : "text",
+            "fields" : {
+                "y" : {
+                    "type" : "foobar",
+                    "fields" : {
+                        "z" : {
+                            "properties" : {
+                                "v" : {
+                                    "type" : "keyword"
+                                },
+                                "w" : {
+                                    "type" : "foo"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
         "some" : {
             "properties" : {
                 "dotted" : {

x-pack/plugin/ql/src/test/resources/mapping-unsupported.json

@@ -6,6 +6,9 @@
         "time_frame" : {
             "type" : "date_range",
             "format" : "yyyy-MM-dd"
+        },
+        "flat" : {
+            "type" : "flattened"
         }
     }
 }

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Analyzer.java

@@ -217,8 +217,14 @@ private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute na
             // unsupported types
             else if (DataTypes.isUnsupported(fa.dataType())) {
                 UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
-                named = u.withUnresolvedMessage(
-                        "Cannot use field [" + fa.name() + "] type [" + unsupportedField.getOriginalType() + "] as is unsupported");
+                if (unsupportedField.hasInherited()) {
+                    named = u.withUnresolvedMessage(
+                            "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "] "
+                                    + "in hierarchy (field [" + unsupportedField.getInherited() + "])");
+                } else {
+                    named = u.withUnresolvedMessage(
+                            "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]");
+                }
             }
             // compound fields
             else if (allowCompound == false && fa.dataType().isPrimitive() == false) {

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java

@@ -482,19 +482,33 @@ public void testGroupByScalarFunctionWithAggOnTarget() {
     }
 
     public void testUnsupportedType() {
-        assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported",
+        assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT unsupported FROM test"));
     }
 
     public void testUnsupportedStarExpansion() {
-        assertEquals("1:8: Cannot use field [unsupported] type [ip_range] as is unsupported",
+        assertEquals("1:8: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT unsupported.* FROM test"));
     }
 
     public void testUnsupportedTypeInFilter() {
-        assertEquals("1:26: Cannot use field [unsupported] type [ip_range] as is unsupported",
+        assertEquals("1:26: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT * FROM test WHERE unsupported > 1"));
     }
+
+    public void testValidRootFieldWithUnsupportedChildren() {
+        accept("SELECT x FROM test");
+    }
+
+    public void testUnsupportedTypeInHierarchy() {
+        assertEquals("1:8: Cannot use field [x.y.z.w] with unsupported type [foobar] in hierarchy (field [y])",
+                error("SELECT x.y.z.w FROM test"));
+        assertEquals("1:8: Cannot use field [x.y.z.v] with unsupported type [foobar] in hierarchy (field [y])",
+                error("SELECT x.y.z.v FROM test"));
+        assertEquals("1:8: Cannot use field [x.y.z] with unsupported type [foobar] in hierarchy (field [y])",
+                error("SELECT x.y.z.* FROM test"));
+        assertEquals("1:8: Cannot use field [x.y] with unsupported type [foobar]", error("SELECT x.y FROM test"));
+    }
 
     public void testTermEqualitOnInexact() {
         assertEquals("1:26: [text = 'value'] cannot operate on first argument field of data type [text]: " +
@@ -509,12 +523,12 @@ public void testTermEqualityOnAmbiguous() {
     }
 
     public void testUnsupportedTypeInFunction() {
-        assertEquals("1:12: Cannot use field [unsupported] type [ip_range] as is unsupported",
+        assertEquals("1:12: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT ABS(unsupported) FROM test"));
     }
 
     public void testUnsupportedTypeInOrder() {
-        assertEquals("1:29: Cannot use field [unsupported] type [ip_range] as is unsupported",
+        assertEquals("1:29: Cannot use field [unsupported] with unsupported type [ip_range]",
                 error("SELECT * FROM test ORDER BY unsupported"));
     }
 

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java

@@ -128,6 +128,94 @@ public void testMetaFieldsAreIgnored() throws Exception {
         assertEquals(DataType.INTEGER, esIndex.mapping().get("_meta_field").getDataType());
         assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
     }
+
+    public void testFlattenedHiddenSubfield() throws Exception {
+        Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
+        addFieldCaps(fieldCaps, "some_field", "flattened", false, false);
+        addFieldCaps(fieldCaps, "some_field._keyed", "flattened", false, false);
+        addFieldCaps(fieldCaps, "another_field", "object", true, false);
+        addFieldCaps(fieldCaps, "another_field._keyed", "keyword", true, false);
+        addFieldCaps(fieldCaps, "nested_field", "object", false, false);
+        addFieldCaps(fieldCaps, "nested_field.sub_field", "flattened", true, true);
+        addFieldCaps(fieldCaps, "nested_field.sub_field._keyed", "flattened", true, true);
+        addFieldCaps(fieldCaps, "text", "keyword", true, true);
+
+        String wildcard = "*";
+        IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
+        assertTrue(resolution.isValid());
+
+        EsIndex esIndex = resolution.get();
+        assertEquals(wildcard, esIndex.name());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getProperties().get("_keyed").getDataType());
+        assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field").getDataType());
+        assertEquals(DataType.UNSUPPORTED,
+                esIndex.mapping().get("nested_field").getProperties().get("sub_field").getProperties().get("_keyed").getDataType());
+        assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
+        assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType());
+        assertEquals(DataType.KEYWORD, esIndex.mapping().get("another_field").getProperties().get("_keyed").getDataType());
+    }
+
+    public void testPropagateUnsupportedTypeToSubFields() throws Exception {
+        // generate a field type having the name of the format "foobar43"
+        String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100);
+        Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
+        addFieldCaps(fieldCaps, "a", "text", false, false);
+        addFieldCaps(fieldCaps, "a.b", esFieldType, false, false);
+        addFieldCaps(fieldCaps, "a.b.c", "object", true, false);
+        addFieldCaps(fieldCaps, "a.b.c.d", "keyword", true, false);
+        addFieldCaps(fieldCaps, "a.b.c.e", "foo", true, true);
+
+        String wildcard = "*";
+        IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
+        assertTrue(resolution.isValid());
+
+        EsIndex esIndex = resolution.get();
+        assertEquals(wildcard, esIndex.name());
+        assertEquals(DataType.TEXT, esIndex.mapping().get("a").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c")
+                .getProperties().get("d").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("a").getProperties().get("b").getProperties().get("c")
+                .getProperties().get("e").getDataType());
+    }
+
+    public void testRandomMappingFieldTypeMappedAsUnsupported() throws Exception {
+        // generate a field type having the name of the format "foobar43"
+        String esFieldType = randomAlphaOfLengthBetween(5, 10) + randomIntBetween(-100, 100);
+
+        Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
+        addFieldCaps(fieldCaps, "some_field", esFieldType, false, false);
+        addFieldCaps(fieldCaps, "another_field", "object", true, false);
+        addFieldCaps(fieldCaps, "another_field._foo", esFieldType, true, false);
+        addFieldCaps(fieldCaps, "nested_field", "object", false, false);
+        addFieldCaps(fieldCaps, "nested_field.sub_field1", esFieldType, true, true);
+        addFieldCaps(fieldCaps, "nested_field.sub_field1.bar", esFieldType, true, true);
+        addFieldCaps(fieldCaps, "nested_field.sub_field2", esFieldType, true, true);
+        // even if this is of a supported type, because it belongs to an UNSUPPORTED type parent, it should also be UNSUPPORTED
+        addFieldCaps(fieldCaps, "nested_field.sub_field2.bar", "keyword", true, true);
+        addFieldCaps(fieldCaps, "text", "keyword", true, true);
+
+        String wildcard = "*";
+        IndexResolution resolution = IndexResolver.mergedMappings(wildcard, new String[] { "index" }, fieldCaps);
+        assertTrue(resolution.isValid());
+
+        EsIndex esIndex = resolution.get();
+        assertEquals(wildcard, esIndex.name());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("some_field").getDataType());
+        assertEquals(DataType.OBJECT, esIndex.mapping().get("nested_field").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getDataType());
+        assertEquals(DataType.UNSUPPORTED,
+                esIndex.mapping().get("nested_field").getProperties().get("sub_field1").getProperties().get("bar").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getDataType());
+        assertEquals(DataType.UNSUPPORTED,
+                esIndex.mapping().get("nested_field").getProperties().get("sub_field2").getProperties().get("bar").getDataType());
+        assertEquals(DataType.KEYWORD, esIndex.mapping().get("text").getDataType());
+        assertEquals(DataType.OBJECT, esIndex.mapping().get("another_field").getDataType());
+        assertEquals(DataType.UNSUPPORTED, esIndex.mapping().get("another_field").getProperties().get("_foo").getDataType());
+    }
 
     public void testMergeIncompatibleCapabilitiesOfObjectFields() throws Exception {
         Map<String, Map<String, FieldCapabilities>> fieldCaps = new HashMap<>();
@@ -330,7 +418,7 @@ private static <K, V> void assertEqualsMaps(Map<K, V> left, Map<K, V> right) {
     private void addFieldCaps(Map<String, Map<String, FieldCapabilities>> fieldCaps, String name, String type, boolean isSearchable,
             boolean isAggregatable) {
         Map<String, FieldCapabilities> cap = new HashMap<>();
-        cap.put(name, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap()));
+        cap.put(type, new FieldCapabilities(name, type, isSearchable, isAggregatable, Collections.emptyMap()));
         fieldCaps.put(name, cap);
     }
 }