Merge branch 'master' into launcher

* master:
  Add a new cluster setting to limit the total number of buckets returned by a request (elastic#27581)
  Allow index settings to be reset by wildcards (elastic#27671)
  Fix UpdateMappingIntegrationIT test failures
  Correct docs for binary fields and their default for doc values (elastic#27680)
  Clarify IntelliJ IDEA Jar Hell fix (elastic#27635)
  Add validation of keystore setting names (elastic#27626)
  Prevent constructing index template without patterns (elastic#27662)
  [DOCS] Fixed typos and broken attribute.
  Add support for filtering mappings fields (elastic#27603)
  [DOCS] Added link to upgrade guide and bumped the upgrade topic up to the top level (elastic#27621)
  [Geo] Add Well Known Text (WKT) Parsing Support to ShapeBuilders
  Fix up tests now that GeoDistance.*.calculate works (elastic#27541)
  [Docs] Fix parameter name (elastic#27656)

CONTRIBUTING.md

@@ -111,19 +111,18 @@ then `File->New Project From Existing Sources`. Point to the root of
 the source directory, select
 `Import project from external model->Gradle`, enable
 `Use auto-import`. In order to run tests directly from
-IDEA 2017.2 and above it is required to disable IDEA run launcher to avoid
-finding yourself in "jar hell", which can be achieved by adding the
+IDEA 2017.2 and above, it is required to disable the IDEA run launcher in order to avoid
+`idea_rt.jar` causing "jar hell". This can be achieved by adding the
 `-Didea.no.launcher=true` [JVM
-option](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties)
-or by adding `idea.no.launcher=true` to the
+option](https://intellij-support.jetbrains.com/hc/en-us/articles/206544869-Configuring-JVM-options-and-platform-properties).
+Alternatively, `idea.no.launcher=true` can be set in the
 [`idea.properties`](https://www.jetbrains.com/help/idea/file-idea-properties.html)
-file which can be accessed under Help > Edit Custom Properties within IDEA. You
-may also need to [remove `ant-javafx.jar` from your
+file which can be accessed under Help > Edit Custom Properties (this will require a
+restart of IDEA). For IDEA 2017.3 and above, in addition to the JVM option, you will need to go to
+`Run->Edit Configurations...` and change the value for the `Shorten command line` setting from
+`user-local default: none` to `classpath file`. You may also need to [remove `ant-javafx.jar` from your
 classpath](https://github.com/elastic/elasticsearch/issues/14348) if that is
-reported as a source of jar hell. Additionally, in order to run tests directly
-from IDEA 2017.3 and above, go to `Run->Edit Configurations...` and change the
-value for the `Shorten command line` setting from `user-local default: none` to
-`classpath file`.
+reported as a source of jar hell.
 
 The Elasticsearch codebase makes heavy use of Java `assert`s and the
 test runner requires that assertions be enabled within the JVM. This

core/src/main/java/org/elasticsearch/ElasticsearchException.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.transport.TcpTransport;
 
 import java.io.IOException;
@@ -986,7 +987,10 @@ private enum ElasticsearchExceptionHandle {
         SHARD_LOCK_OBTAIN_FAILED_EXCEPTION(org.elasticsearch.env.ShardLockObtainFailedException.class,
                                            org.elasticsearch.env.ShardLockObtainFailedException::new, 147, Version.V_5_0_2),
         UNKNOWN_NAMED_OBJECT_EXCEPTION(org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException.class,
-                org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException::new, 148, Version.V_5_2_0);
+                org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException::new, 148, Version.V_5_2_0),
+        TOO_MANY_BUCKETS_EXCEPTION(MultiBucketConsumerService.TooManyBucketsException.class,
+                                   MultiBucketConsumerService.TooManyBucketsException::new, 149,
+            Version.V_7_0_0_alpha1);
 
         final Class<? extends ElasticsearchException> exceptionClass;
         final CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> constructor;

core/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java

@@ -36,20 +36,27 @@
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
 import java.util.List;
 
 /**
  * Get index action.
  */
 public class TransportGetIndexAction extends TransportClusterInfoAction<GetIndexRequest, GetIndexResponse> {
 
+    private final IndicesService indicesService;
+
     @Inject
     public TransportGetIndexAction(Settings settings, TransportService transportService, ClusterService clusterService,
-                                   ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, GetIndexAction.NAME, transportService, clusterService, threadPool, actionFilters, GetIndexRequest::new, indexNameExpressionResolver);
+                                   ThreadPool threadPool, ActionFilters actionFilters,
+                                   IndexNameExpressionResolver indexNameExpressionResolver, IndicesService indicesService) {
+        super(settings, GetIndexAction.NAME, transportService, clusterService, threadPool, actionFilters, GetIndexRequest::new,
+                indexNameExpressionResolver);
+        this.indicesService = indicesService;
     }
 
     @Override
@@ -60,7 +67,8 @@ protected String executor() {
 
     @Override
     protected ClusterBlockException checkBlock(GetIndexRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
+                indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
@@ -82,8 +90,14 @@ protected void doMasterOperation(final GetIndexRequest request, String[] concret
             switch (feature) {
             case MAPPINGS:
                     if (!doneMappings) {
-                        mappingsResult = state.metaData().findMappings(concreteIndices, request.types());
-                        doneMappings = true;
+                        try {
+                            mappingsResult = state.metaData().findMappings(concreteIndices, request.types(),
+                                    indicesService.getFieldFilter());
+                            doneMappings = true;
+                        } catch (IOException e) {
+                            listener.onFailure(e);
+                            return;
+                        }
                     }
                     break;
             case ALIASES:

core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java

@@ -29,13 +29,12 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.routing.ShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
-import org.elasticsearch.common.collect.MapBuilder;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.DocumentFieldMappers;
@@ -50,7 +49,10 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 import static java.util.Collections.singletonMap;
@@ -69,7 +71,8 @@ public class TransportGetFieldMappingsIndexAction extends TransportSingleShardAc
     public TransportGetFieldMappingsIndexAction(Settings settings, ClusterService clusterService, TransportService transportService,
                                                 IndicesService indicesService, ThreadPool threadPool, ActionFilters actionFilters,
                                                 IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, GetFieldMappingsIndexRequest::new, ThreadPool.Names.MANAGEMENT);
+        super(settings, ACTION_NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver,
+                GetFieldMappingsIndexRequest::new, ThreadPool.Names.MANAGEMENT);
         this.clusterService = clusterService;
         this.indicesService = indicesService;
     }
@@ -90,6 +93,9 @@ protected ShardsIterator shards(ClusterState state, InternalRequest request) {
     protected GetFieldMappingsResponse shardOperation(final GetFieldMappingsIndexRequest request, ShardId shardId) {
         assert shardId != null;
         IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
+        Predicate<String> metadataFieldPredicate = indicesService::isMetaDataField;
+        Predicate<String> fieldPredicate = metadataFieldPredicate.or(indicesService.getFieldFilter().apply(shardId.getIndexName()));
+
         Collection<String> typeIntersection;
         if (request.types().length == 0) {
             typeIntersection = indexService.mapperService().types();
@@ -104,16 +110,15 @@ protected GetFieldMappingsResponse shardOperation(final GetFieldMappingsIndexReq
             }
         }
 
-        MapBuilder<String, Map<String, FieldMappingMetaData>> typeMappings = new MapBuilder<>();
+        Map<String, Map<String, FieldMappingMetaData>> typeMappings = new HashMap<>();
         for (String type : typeIntersection) {
             DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
-            Map<String, FieldMappingMetaData> fieldMapping = findFieldMappingsByType(documentMapper, request);
+            Map<String, FieldMappingMetaData> fieldMapping = findFieldMappingsByType(fieldPredicate, documentMapper, request);
             if (!fieldMapping.isEmpty()) {
                 typeMappings.put(type, fieldMapping);
             }
         }
-
-        return new GetFieldMappingsResponse(singletonMap(shardId.getIndexName(), typeMappings.immutableMap()));
+        return new GetFieldMappingsResponse(singletonMap(shardId.getIndexName(), Collections.unmodifiableMap(typeMappings)));
     }
 
     @Override
@@ -163,47 +168,50 @@ public Boolean paramAsBoolean(String key, Boolean defaultValue) {
         }
     };
 
-    private Map<String, FieldMappingMetaData> findFieldMappingsByType(DocumentMapper documentMapper, GetFieldMappingsIndexRequest request) {
-        MapBuilder<String, FieldMappingMetaData> fieldMappings = new MapBuilder<>();
+    private static Map<String, FieldMappingMetaData> findFieldMappingsByType(Predicate<String> fieldPredicate,
+                                                                             DocumentMapper documentMapper,
+                                                                             GetFieldMappingsIndexRequest request) {
+        Map<String, FieldMappingMetaData> fieldMappings = new HashMap<>();
         final DocumentFieldMappers allFieldMappers = documentMapper.mappers();
         for (String field : request.fields()) {
             if (Regex.isMatchAllPattern(field)) {
                 for (FieldMapper fieldMapper : allFieldMappers) {
-                    addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
+                    addFieldMapper(fieldPredicate, fieldMapper.fieldType().name(), fieldMapper, fieldMappings, request.includeDefaults());
                 }
             } else if (Regex.isSimpleMatchPattern(field)) {
                 for (FieldMapper fieldMapper : allFieldMappers) {
                     if (Regex.simpleMatch(field, fieldMapper.fieldType().name())) {
-                        addFieldMapper(fieldMapper.fieldType().name(), fieldMapper, fieldMappings,
-                            request.includeDefaults());
+                        addFieldMapper(fieldPredicate,  fieldMapper.fieldType().name(),
+                                fieldMapper, fieldMappings, request.includeDefaults());
                     }
                 }
             } else {
                 // not a pattern
                 FieldMapper fieldMapper = allFieldMappers.smartNameFieldMapper(field);
                 if (fieldMapper != null) {
-                    addFieldMapper(field, fieldMapper, fieldMappings, request.includeDefaults());
+                    addFieldMapper(fieldPredicate, field, fieldMapper, fieldMappings, request.includeDefaults());
                 } else if (request.probablySingleFieldRequest()) {
                     fieldMappings.put(field, FieldMappingMetaData.NULL);
                 }
             }
         }
-        return fieldMappings.immutableMap();
+        return Collections.unmodifiableMap(fieldMappings);
     }
 
-    private void addFieldMapper(String field, FieldMapper fieldMapper, MapBuilder<String, FieldMappingMetaData> fieldMappings, boolean includeDefaults) {
+    private static void addFieldMapper(Predicate<String> fieldPredicate,
+                                       String field, FieldMapper fieldMapper, Map<String, FieldMappingMetaData> fieldMappings,
+                                       boolean includeDefaults) {
         if (fieldMappings.containsKey(field)) {
             return;
         }
-        try {
-            XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
-            builder.startObject();
-            fieldMapper.toXContent(builder, includeDefaults ? includeDefaultsParams : ToXContent.EMPTY_PARAMS);
-            builder.endObject();
-            fieldMappings.put(field, new FieldMappingMetaData(fieldMapper.fieldType().name(), builder.bytes()));
-        } catch (IOException e) {
-            throw new ElasticsearchException("failed to serialize XContent of field [" + field + "]", e);
+        if (fieldPredicate.test(field)) {
+            try {
+                BytesReference bytes = XContentHelper.toXContent(fieldMapper, XContentType.JSON,
+                        includeDefaults ? includeDefaultsParams : ToXContent.EMPTY_PARAMS, false);
+                fieldMappings.put(field, new FieldMappingMetaData(fieldMapper.fieldType().name(), bytes));
+            } catch (IOException e) {
+                throw new ElasticsearchException("failed to serialize XContent of field [" + field + "]", e);
+            }
         }
     }
-
 }

core/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java

@@ -31,15 +31,23 @@
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.io.IOException;
+
 public class TransportGetMappingsAction extends TransportClusterInfoAction<GetMappingsRequest, GetMappingsResponse> {
 
+    private final IndicesService indicesService;
+
     @Inject
     public TransportGetMappingsAction(Settings settings, TransportService transportService, ClusterService clusterService,
-                                      ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) {
-        super(settings, GetMappingsAction.NAME, transportService, clusterService, threadPool, actionFilters, GetMappingsRequest::new, indexNameExpressionResolver);
+                                      ThreadPool threadPool, ActionFilters actionFilters,
+                                      IndexNameExpressionResolver indexNameExpressionResolver, IndicesService indicesService) {
+        super(settings, GetMappingsAction.NAME, transportService, clusterService, threadPool, actionFilters, GetMappingsRequest::new,
+                indexNameExpressionResolver);
+        this.indicesService = indicesService;
     }
 
     @Override
@@ -50,7 +58,8 @@ protected String executor() {
 
     @Override
     protected ClusterBlockException checkBlock(GetMappingsRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request));
+        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ,
+                indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override
@@ -59,11 +68,15 @@ protected GetMappingsResponse newResponse() {
     }
 
     @Override
-    protected void doMasterOperation(final GetMappingsRequest request, String[] concreteIndices, final ClusterState state, final ActionListener<GetMappingsResponse> listener) {
+    protected void doMasterOperation(final GetMappingsRequest request, String[] concreteIndices, final ClusterState state,
+                                     final ActionListener<GetMappingsResponse> listener) {
         logger.trace("serving getMapping request based on version {}", state.version());
-        ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> result = state.metaData().findMappings(
-                concreteIndices, request.types()
-        );
-        listener.onResponse(new GetMappingsResponse(result));
+        try {
+            ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> result =
+                    state.metaData().findMappings(concreteIndices, request.types(), indicesService.getFieldFilter());
+            listener.onResponse(new GetMappingsResponse(result));
+        } catch (IOException e) {
+            listener.onFailure(e);
+        }
     }
 }

core/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java

@@ -40,6 +40,7 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Predicate;
 
 public class TransportFieldCapabilitiesIndexAction extends TransportSingleShardAction<FieldCapabilitiesIndexRequest,
     FieldCapabilitiesIndexResponse> {
@@ -77,12 +78,15 @@ protected FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesI
         for (String field : request.fields()) {
             fieldNames.addAll(mapperService.simpleMatchToIndexNames(field));
         }
+        Predicate<String> fieldPredicate = indicesService.getFieldFilter().apply(shardId.getIndexName());
         Map<String, FieldCapabilities> responseMap = new HashMap<>();
         for (String field : fieldNames) {
             MappedFieldType ft = mapperService.fullName(field);
             if (ft != null) {
                 FieldCapabilities fieldCap = new FieldCapabilities(field, ft.typeName(), ft.isSearchable(), ft.isAggregatable());
-                responseMap.put(field, fieldCap);
+                if (indicesService.isMetaDataField(field) || fieldPredicate.test(field)) {
+                    responseMap.put(field, fieldCap);
+                }
             }
         }
         return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), responseMap);