Using FieldTypeMapper to map physical types onto logical types. (Gray…

…log2/graylog-plugin-enterprise#214)

* Using FieldTypeMapper to map physical types onto logical types.

This change employs the `FieldTypeMapper` class in the
`FieldTypesResource` to map ES types to Graylog types before returning
field types. The frontend code is adapted to the structure which is
returned. As an example, the "Chart" action eligibility check is changed
from checking the field name to checking for the `numeric` property in
the field type.

* Object is not an immutable map anymore.

* Updating yarn.lock.

enterprise/src/main/java/org/graylog/plugins/enterprise/search/rest/FieldTypesResource.java

@@ -1,9 +1,13 @@
 package org.graylog.plugins.enterprise.search.rest;
 
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
 import io.swagger.annotations.Api;
 import org.apache.shiro.authz.annotation.RequiresAuthentication;
 import org.graylog2.database.NotFoundException;
 import org.graylog2.indexer.fieldtypes.FieldTypeDTO;
+import org.graylog2.indexer.fieldtypes.FieldTypeMapper;
+import org.graylog2.indexer.fieldtypes.FieldTypes;
 import org.graylog2.indexer.fieldtypes.IndexFieldTypesDTO;
 import org.graylog2.indexer.fieldtypes.IndexFieldTypesService;
 import org.graylog2.plugin.rest.PluginRestResource;
@@ -22,47 +26,65 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static com.google.common.collect.ImmutableSet.of;
+import static org.graylog2.indexer.fieldtypes.FieldTypes.Type.createType;
+
 @Api(value = "Enterprise/Field Types", description = "Field Types")
 @Path("/fields")
 @Produces(MediaType.APPLICATION_JSON)
 @RequiresAuthentication
 public class FieldTypesResource extends RestResource implements PluginRestResource {
     private final IndexFieldTypesService indexFieldTypesService;
     private final StreamService streamService;
+    private final FieldTypeMapper fieldTypeMapper;
+    private static final FieldTypes.Type UNKNOWN_TYPE = createType("unknown", of());
 
     @Inject
-    public FieldTypesResource(IndexFieldTypesService indexFieldTypesService, StreamService streamService) {
+    public FieldTypesResource(IndexFieldTypesService indexFieldTypesService, StreamService streamService, FieldTypeMapper fieldTypeMapper) {
         this.indexFieldTypesService = indexFieldTypesService;
         this.streamService = streamService;
+        this.fieldTypeMapper = fieldTypeMapper;
     }
 
-    private Set<FieldTypeDTO> mergeCompoundFieldTypes(Stream<FieldTypeDTO> stream) {
-        return stream.collect(Collectors.groupingBy(FieldTypeDTO::fieldName, Collectors.toSet()))
+    private Set<MappedFieldTypeDTO> mergeCompoundFieldTypes(Stream<MappedFieldTypeDTO> stream) {
+        return stream.collect(Collectors.groupingBy(MappedFieldTypeDTO::name, Collectors.toSet()))
                 .entrySet()
                 .stream()
                 .map(entry -> {
-                    final Set<FieldTypeDTO> fieldTypes = entry.getValue();
+                    final Set<MappedFieldTypeDTO> fieldTypes = entry.getValue();
                     final String fieldName = entry.getKey();
                     if (fieldTypes.size() == 1) {
                         return fieldTypes.iterator().next();
                     }
-                    final String compoundFieldType = "compound(" + fieldTypes.stream().map(FieldTypeDTO::physicalType).collect(Collectors.joining(",")) + ")";
-                    return FieldTypeDTO.create(fieldName, compoundFieldType);
+                    final String compoundFieldType = "compound(" + fieldTypes.stream()
+                            .map(mappedFieldTypeDTO -> mappedFieldTypeDTO.type().type())
+                            .collect(Collectors.joining(",")) + ")";
+                    final ImmutableSet<String> commonProperties = fieldTypes.stream()
+                            .map(mappedFieldTypeDTO -> mappedFieldTypeDTO.type().properties())
+                            .reduce((s1, s2) -> Sets.intersection(s1, s2).immutableCopy())
+                            .orElse(ImmutableSet.of());
+                    return MappedFieldTypeDTO.create(fieldName, createType(compoundFieldType, commonProperties));
                 })
                 .collect(Collectors.toSet());
 
     }
 
+    private MappedFieldTypeDTO mapPhysicalFieldType(FieldTypeDTO fieldType) {
+        final FieldTypes.Type mappedFieldType = fieldTypeMapper.mapType(fieldType.physicalType()).orElse(UNKNOWN_TYPE);
+        return MappedFieldTypeDTO.create(fieldType.fieldName(), mappedFieldType);
+    }
+
     @GET
-    public Set<FieldTypeDTO> allFieldTypes() {
+    public Set<MappedFieldTypeDTO> allFieldTypes() {
         return mergeCompoundFieldTypes(indexFieldTypesService.findAll()
                 .stream()
                 .map(IndexFieldTypesDTO::fields)
-                .flatMap(Collection::stream));
+                .flatMap(Collection::stream)
+                .map(this::mapPhysicalFieldType));
     }
 
     @POST
-    public Set<FieldTypeDTO> byStreams(FieldTypesForStreamsRequest request) {
+    public Set<MappedFieldTypeDTO> byStreams(FieldTypesForStreamsRequest request) {
         return mergeCompoundFieldTypes(request.streams()
                 .stream()
                 .map(streamId -> {
@@ -76,6 +98,7 @@ public Set<FieldTypeDTO> byStreams(FieldTypesForStreamsRequest request) {
                 .map(indexSet -> indexSet.getIndexSet().getConfig().id())
                 .flatMap(indexSetId -> this.indexFieldTypesService.findForIndexSet(indexSetId).stream())
                 .map(IndexFieldTypesDTO::fields)
-                .flatMap(Collection::stream));
+                .flatMap(Collection::stream)
+                .map(this::mapPhysicalFieldType));
     }
 }

enterprise/src/main/java/org/graylog/plugins/enterprise/search/rest/MappedFieldTypeDTO.java

@@ -0,0 +1,22 @@
+package org.graylog.plugins.enterprise.search.rest;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+import org.graylog2.indexer.fieldtypes.FieldTypes;
+
+@AutoValue
+@JsonAutoDetect
+public abstract class MappedFieldTypeDTO {
+    @JsonProperty("name")
+    public abstract String name();
+
+    @JsonProperty("type")
+    public abstract FieldTypes.Type type();
+
+    @JsonCreator
+    public static MappedFieldTypeDTO create(@JsonProperty("name") String name, @JsonProperty("type") FieldTypes.Type type) {
+        return new AutoValue_MappedFieldTypeDTO(name, type);
+    }
+}

enterprise/src/web/enterprise/bindings.jsx

@@ -91,7 +91,7 @@ export default {
       type: 'chart',
       title: 'Chart',
       handler: ChartActionHandler,
-      condition: ({ type }) => type === 'long',
+      condition: ({ type }) => type.isNumeric(),
     },
     {
       type: 'aggregate',

enterprise/src/web/enterprise/components/Field.jsx

@@ -7,6 +7,7 @@ import { PluginStore } from 'graylog-web-plugin/plugin';
 import OverlayDropdown from './OverlayDropdown';
 
 import style from './Field.css';
+import FieldType from '../logic/fieldtypes/FieldType';
 
 export default class Field extends React.Component {
   static propTypes = {
@@ -15,8 +16,7 @@ export default class Field extends React.Component {
     name: PropTypes.string.isRequired,
     menuContainer: PropTypes.object,
     queryId: PropTypes.string.isRequired,
-    type: PropTypes.string.isRequired,
-    viewId: PropTypes.string,
+    type: PropTypes.instanceOf(FieldType).isRequired,
   };
 
   static defaultProps = {
@@ -37,7 +37,7 @@ export default class Field extends React.Component {
   _onMenuToggle = () => this.setState(state => ({ open: !state.open }));
 
   render() {
-    const { children, disabled, menuContainer, name, queryId, type, viewId } = this.props;
+    const { children, disabled, menuContainer, name, queryId, type } = this.props;
     const element = children || name;
     const wrappedElement = disabled ? <span className={style.disabled}>{element}</span> : element;
     const fieldActions = PluginStore.exports('fieldActions').map((fieldAction) => {
@@ -60,7 +60,7 @@ export default class Field extends React.Component {
                        menuContainer={menuContainer} >
         <div style={{ marginBottom: '10px' }}>
           <span className={style.dropdownheader}>
-            {name} = {type}
+            {name} = {type.type}
           </span>
         </div>
 

enterprise/src/web/enterprise/components/aggregationbuilder/AggregationControls.jsx

@@ -67,7 +67,7 @@ export default class AggregationControls extends React.Component {
     const { children, fields } = this.props;
     const { columnPivots, rowPivots, series, sort, visualization } = this.state.config.toObject();
     const formattedFields = fields
-      .map(fieldType => fieldType.get('field_name'))
+      .map(fieldType => fieldType.name)
       .map(v => ({ label: v, value: v }))
       .valueSeq()
       .toJS()

enterprise/src/web/enterprise/components/aggregationbuilder/ConfigurablePivot.jsx

@@ -9,6 +9,7 @@ import connect from 'stores/connect';
 
 import PivotConfiguration from './PivotConfiguration';
 import { FieldTypesStore } from '../../stores/FieldTypesStore';
+import FieldType from '../../logic/fieldtypes/FieldType';
 
 class ConfigurablePivot extends React.Component {
   static propTypes = {};
@@ -31,7 +32,8 @@ class ConfigurablePivot extends React.Component {
 
   render() {
     const { value, config, fields } = this.props;
-    const fieldType = fields.all.filter(v => v.get('field_name') === value.label).getIn([0, 'physical_type'], 'unknown');
+    const fieldTypes = fields.all.filter(v => v.name === value.label);
+    const fieldType = fieldTypes.isEmpty() ? FieldType.Unknown : fieldTypes.first().type;
     const popover = this.state.isOpen && (
       <Portal>
         <Position

enterprise/src/web/enterprise/components/aggregationbuilder/PivotConfiguration.jsx

@@ -4,17 +4,18 @@ import PropTypes from 'prop-types';
 import { Button } from 'react-bootstrap';
 import TimeHistogramPivot from './pivottypes/TimeHistogramPivot';
 import NoConfigurationPivot from './pivottypes/NoConfigurationPivot';
+import FieldType from '../../logic/fieldtypes/FieldType';
 
 const _configurationComponentByType = (type, value, onChange) => {
-  switch (type) {
+  switch (type.type) {
     case 'date': return <TimeHistogramPivot onChange={onChange} value={value} />;
     default: return <NoConfigurationPivot />;
   }
 };
 
 export default class PivotConfiguration extends React.Component {
   static propTypes = {
-    type: PropTypes.string.isRequired,
+    type: PropTypes.instanceOf(FieldType).isRequired,
     config: PropTypes.shape({
       type: PropTypes.string.isRequired,
     }).isRequired,

enterprise/src/web/enterprise/components/messagelist/MessageFields.jsx

@@ -3,6 +3,7 @@ import React from 'react';
 
 import { ChangedMessageField } from 'components/search';
 import { MessageField } from 'enterprise/components/messagelist';
+import FieldType from 'enterprise/logic/fieldtypes/FieldType';
 
 class MessageFields extends React.Component {
   static propTypes = {
@@ -25,18 +26,23 @@ class MessageFields extends React.Component {
       return Object.keys(fields)
         .sort()
         .map((key) => {
-          const fieldType = this.props.fields.find(type => type.get('field_name') === key);
+          const fieldTypeMapping = this.props.fields.find(type => type.name === key);
+          const fieldType = fieldTypeMapping ? fieldTypeMapping.type : FieldType.Unknown;
           return (
-            <MessageField key={key} {...this.props} fieldName={key} value={fields[key]} fieldType={fieldType ? fieldType.get('physical_type') : 'unknown'}
-                               disableFieldActions={this.props.disableFieldActions || decoratedFields.indexOf(key) !== -1} />
+            <MessageField key={key}
+                          {...this.props}
+                          fieldName={key}
+                          value={fields[key]}
+                          fieldType={fieldType}
+                          disableFieldActions={this.props.disableFieldActions || decoratedFields.indexOf(key) !== -1} />
           );
         });
     }
 
     const allKeys = Object.keys(decorationStats.removed_fields).concat(Object.keys(fields)).sort();
 
     return allKeys.map((key) => {
-      const fieldType = this.props.fields.find(f => f.get('field_name') === key);
+      const fieldType = this.props.fields.find(f => f.name === key) || { type: FieldType.Unknown };
       if (decorationStats.added_fields[key]) {
         return <ChangedMessageField key={key} fieldName={key} newValue={fields[key]} />;
       }
@@ -53,7 +59,7 @@ class MessageFields extends React.Component {
                                      originalValue={decorationStats.removed_fields[key]} />);
       }
 
-      return <MessageField key={key} {...this.props} fieldName={key} fieldType={fieldType.get('physical_type')} value={fields[key]} disableFieldActions />;
+      return <MessageField key={key} {...this.props} fieldName={key} fieldType={fieldType.type} value={fields[key]} disableFieldActions />;
     });
   };
 

enterprise/src/web/enterprise/components/messagelist/StaticMessageList.jsx

@@ -7,6 +7,7 @@ import connect from 'stores/connect';
 import WidgetHeader from '../widgets/WidgetHeader';
 import MessageList from '../widgets/MessageList';
 import { FieldTypesStore } from '../../stores/FieldTypesStore';
+import FieldTypeMapping from '../../logic/fieldtypes/FieldTypeMapping';
 
 const StaticMessageList = ({ showMessages, onToggleMessages, fieldTypes, messages }) => (
   <div className={style.widgetContainer}>
@@ -21,7 +22,7 @@ const StaticMessageList = ({ showMessages, onToggleMessages, fieldTypes, message
 
 StaticMessageList.propTypes = {
   fieldTypes: PropTypes.shape({
-    all: ImmutablePropTypes.listOf(ImmutablePropTypes.map),
+    all: ImmutablePropTypes.listOf(PropTypes.instanceOf(FieldTypeMapping)),
   }).isRequired,
   messages: PropTypes.shape({
     messages: PropTypes.arrayOf(PropTypes.object).isRequired,

enterprise/src/web/enterprise/components/searchbar/SearchBarAutocompletions.js

@@ -3,18 +3,17 @@ import { FieldTypesStore } from 'enterprise/stores/FieldTypesStore';
 import { ViewMetadataStore } from '../../stores/ViewMetadataStore';
 
 const _fieldResult = (field, score = 1) => {
-  // eslint-disable-next-line camelcase
-  const { field_name, physical_type } = field.toObject();
+  const { name, type } = field;
   return {
-    name: field_name,
-    value: field_name,
+    name: name,
+    value: name,
     score,
-    meta: physical_type,
+    meta: type.type,
   };
 };
 
 const _matchesFieldName = (prefix) => {
-  return field => (field.get('field_name').indexOf(prefix) >= 0);
+  return field => (field.name.indexOf(prefix) >= 0);
 };
 
 export default class SearchBarAutoCompletions {

enterprise/src/web/enterprise/components/sidebar/FieldList.jsx

@@ -53,9 +53,8 @@ const FieldList = createReactClass({
   },
 
   _renderField({ fields, fieldType, selectedQuery, selectedView, selectedFields }) {
-    const name = fieldType.get('field_name');
-    const type = fieldType.get('physical_type');
-    const disabled = !fields.find(f => f.get('field_name') === name);
+    const { name, type } = fieldType;
+    const disabled = !fields.find(f => f.name === name);
 
     return (
       <li key={`field-${name}`} className={styles.fieldListItem} >

enterprise/src/web/enterprise/components/sidebar/FieldTypeIcon.jsx

@@ -2,14 +2,18 @@ import React from 'react';
 import PropTypes from 'prop-types';
 
 import styles from './FieldTypeIcon.css';
+import FieldType from '../../logic/fieldtypes/FieldType';
 
 const iconClass = (type) => {
   switch (type) {
-    case 'keyword':
+    case 'string':
       return 'font';
-    case 'text':
-      return 'paragraph';
+    case 'byte':
+    case 'double':
+    case 'float':
+    case 'int':
     case 'long':
+    case 'short':
       return 'line-chart';
     case 'date':
       return 'calendar';
@@ -18,11 +22,11 @@ const iconClass = (type) => {
   }
 };
 const FieldTypeIcon = ({ type }) => {
-  return <i className={`fa fa-${iconClass(type)} ${styles.fieldTypeIcon}`} />;
+  return <i className={`fa fa-${iconClass(type.type)} ${styles.fieldTypeIcon}`} />;
 };
 
 FieldTypeIcon.propTypes = {
-  type: PropTypes.string.isRequired,
+  type: PropTypes.instanceOf(FieldType).isRequired,
 };
 
 export default FieldTypeIcon;

enterprise/src/web/enterprise/components/widgets/MessageList.jsx

@@ -14,6 +14,7 @@ import { QueriesStore } from 'enterprise/stores/QueriesStore';
 import { SelectedFieldsStore } from '../../stores/SelectedFieldsStore';
 import { ViewStore } from '../../stores/ViewStore';
 import { SearchConfigStore } from '../../stores/SearchConfigStore';
+import FieldType from '../../logic/fieldtypes/FieldType';
 
 const RefreshActions = ActionsProvider.getActions('Refresh');
 
@@ -90,7 +91,7 @@ const MessageList = createReactClass({
   },
 
   _fieldTypeFor(fieldName, fields) {
-    return fields.find(f => f.get('field_name') === fieldName).get('physical_type', 'unknown')
+    return (fields.find(f => f.name === fieldName) || { type: FieldType.Unknown }).type;
   },
 
   render() {