Merge branch 'master' into docgen

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -130,13 +130,6 @@ class BuildPlugin implements Plugin<Project> {
             String runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome)
             File gradleJavaHome = Jvm.current().javaHome
 
-            final Map<Integer, String> javaVersions = [:]
-            for (int version = 8; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) {
-                if(System.getenv(getJavaHomeEnvVarName(version.toString())) != null) {
-                    javaVersions.put(version, findJavaHome(version.toString()));
-                }
-            }
-
             String javaVendor = System.getProperty('java.vendor')
             String gradleJavaVersion = System.getProperty('java.version')
             String gradleJavaVersionDetails = "${javaVendor} ${gradleJavaVersion}" +
@@ -197,38 +190,48 @@ class BuildPlugin implements Plugin<Project> {
                 throw new GradleException(message)
             }
 
-            ExecutorService exec = Executors.newFixedThreadPool(javaVersions.size())
-            Set<Future<Void>> results = new HashSet<>()
-
-            javaVersions.entrySet().stream()
-                    .filter { it.getValue() != null }
-                    .forEach { javaVersionEntry ->
-                        results.add(exec.submit {
-                            final String javaHome = javaVersionEntry.getValue()
-                            final int version = javaVersionEntry.getKey()
-                            if (project.file(javaHome).exists() == false) {
-                                throw new GradleException("Invalid JAVA${version}_HOME=${javaHome} location does not exist")
-                            }
+            final Map<Integer, String> javaVersions = [:]
+            for (int version = 8; version <= Integer.parseInt(minimumCompilerVersion.majorVersion); version++) {
+                if(System.getenv(getJavaHomeEnvVarName(version.toString())) != null) {
+                    javaVersions.put(version, findJavaHome(version.toString()));
+                }
+            }
 
-                            JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, javaHome))
-                            final JavaVersion expectedJavaVersionEnum = version < 9 ?
-                                    JavaVersion.toVersion("1." + version) :
-                                    JavaVersion.toVersion(Integer.toString(version))
+            if (javaVersions.isEmpty() == false) {
 
-                            if (javaVersionEnum != expectedJavaVersionEnum) {
-                                final String message =
-                                        "the environment variable JAVA" + version + "_HOME must be set to a JDK installation directory for Java" +
-                                                " ${expectedJavaVersionEnum} but is [${javaHome}] corresponding to [${javaVersionEnum}]"
-                                throw new GradleException(message)
-                            }
-                        })
-            }
+                ExecutorService exec = Executors.newFixedThreadPool(javaVersions.size())
+                Set<Future<Void>> results = new HashSet<>()
+
+                javaVersions.entrySet().stream()
+                        .filter { it.getValue() != null }
+                        .forEach { javaVersionEntry ->
+                    results.add(exec.submit {
+                        final String javaHome = javaVersionEntry.getValue()
+                        final int version = javaVersionEntry.getKey()
+                        if (project.file(javaHome).exists() == false) {
+                            throw new GradleException("Invalid JAVA${version}_HOME=${javaHome} location does not exist")
+                        }
 
-            project.gradle.taskGraph.whenReady {
-                try {
-                    results.forEach { it.get() }
-                } finally {
-                    exec.shutdown();
+                        JavaVersion javaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(project, javaHome))
+                        final JavaVersion expectedJavaVersionEnum = version < 9 ?
+                                JavaVersion.toVersion("1." + version) :
+                                JavaVersion.toVersion(Integer.toString(version))
+
+                        if (javaVersionEnum != expectedJavaVersionEnum) {
+                            final String message =
+                                    "the environment variable JAVA" + version + "_HOME must be set to a JDK installation directory for Java" +
+                                            " ${expectedJavaVersionEnum} but is [${javaHome}] corresponding to [${javaVersionEnum}]"
+                            throw new GradleException(message)
+                        }
+                    })
+                }
+
+                project.gradle.taskGraph.whenReady {
+                    try {
+                        results.forEach { it.get() }
+                    } finally {
+                        exec.shutdown();
+                    }
                 }
             }
 

distribution/archives/build.gradle

@@ -113,25 +113,25 @@ task buildIntegTestZip(type: Zip) {
 task buildWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(false), 'zip', 'windows', false, true)
+  with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', false, true)
 }
 
 task buildOssWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'windows-x86_64'
-  with archiveFiles(modulesFiles(true), 'zip', 'windows', true, true)
+  with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', true, true)
 }
 
 task buildNoJdkWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'no-jdk-windows-x86_64'
-  with archiveFiles(modulesFiles(false), 'zip', 'windows', false, false)
+  with archiveFiles(modulesFiles(false, 'windows'), 'zip', 'windows', false, false)
 }
 
 task buildOssNoJdkWindowsZip(type: Zip) {
   configure(commonZipConfig)
   archiveClassifier = 'no-jdk-windows-x86_64'
-  with archiveFiles(modulesFiles(true), 'zip', 'windows', true, false)
+  with archiveFiles(modulesFiles(true, 'windows'), 'zip', 'windows', true, false)
 }
 
 Closure commonTarConfig = {
@@ -144,49 +144,49 @@ Closure commonTarConfig = {
 task buildDarwinTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(false), 'tar', 'darwin', false, true)
+  with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', false, true)
 }
 
 task buildOssDarwinTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'darwin-x86_64'
-  with archiveFiles(modulesFiles(true), 'tar', 'darwin', true, true)
+  with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', true, true)
 }
 
 task buildNoJdkDarwinTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-darwin-x86_64'
-  with archiveFiles(modulesFiles(false), 'tar', 'darwin', false, false)
+  with archiveFiles(modulesFiles(false, 'darwin'), 'tar', 'darwin', false, false)
 }
 
 task buildOssNoJdkDarwinTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-darwin-x86_64'
-  with archiveFiles(modulesFiles(true), 'tar', 'darwin', true, false)
+  with archiveFiles(modulesFiles(true, 'darwin'), 'tar', 'darwin', true, false)
 }
 
 task buildLinuxTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(false), 'tar', 'linux', false, true)
+  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', false, true)
 }
 
 task buildOssLinuxTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'linux-x86_64'
-  with archiveFiles(modulesFiles(true), 'tar', 'linux', true, true)
+  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', true, true)
 }
 
 task buildNoJdkLinuxTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-linux-x86_64'
-  with archiveFiles(modulesFiles(false), 'tar', 'linux', false, false)
+  with archiveFiles(modulesFiles(false, 'linux'), 'tar', 'linux', false, false)
 }
 
 task buildOssNoJdkLinuxTar(type: Tar) {
   configure(commonTarConfig)
   archiveClassifier = 'no-jdk-linux-x86_64'
-  with archiveFiles(modulesFiles(true), 'tar', 'linux', true, false)
+  with archiveFiles(modulesFiles(true, 'linux'), 'tar', 'linux', true, false)
 }
 
 Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }

distribution/build.gradle

@@ -296,6 +296,9 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           from { project(':distribution:tools:plugin-cli').jar }
           from { project(':distribution:tools:plugin-cli').configurations.runtime }
         }
+        into('tools/keystore-cli') {
+          from { project(':distribution:tools:keystore-cli').jar }
+        }
         if (oss == false) {
           into('tools/security-cli') {
             from { project(':x-pack:plugin:security:cli').jar }
@@ -305,7 +308,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       }
     }
 
-    modulesFiles = { oss ->
+    modulesFiles = { oss, platform ->
       copySpec {
         eachFile {
           if (it.relativePath.segments[-2] == 'bin') {
@@ -315,10 +318,22 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
             it.mode = 0644
           }
         }
+        Task buildModules
         if (oss) {
-          from project(':distribution').buildOssModules
+          buildModules = project(':distribution').buildOssModules
+        } else {
+          buildModules = project(':distribution').buildDefaultModules
+        }
+        List excludePlatforms = ['linux', 'windows', 'darwin']
+        if (platform != null) {
+           excludePlatforms.remove(excludePlatforms.indexOf(platform))
         } else {
-          from project(':distribution').buildDefaultModules
+           excludePlatforms = []
+        }
+        from(buildModules) {
+          for (String excludePlatform : excludePlatforms) {
+            exclude "**/platform/${excludePlatform}-x86_64/**"
+          }
         }
       }
     }

distribution/packages/build.gradle

@@ -139,7 +139,7 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) {
         with libFiles(oss)
       }
       into('modules') {
-        with modulesFiles(oss)
+        with modulesFiles(oss, 'linux')
       }
       if (jdk) {
         into('jdk') {

distribution/src/bin/elasticsearch-keystore

@@ -1,5 +1,6 @@
 #!/bin/bash
 
 ES_MAIN_CLASS=org.elasticsearch.common.settings.KeyStoreCli \
+  ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/keystore-cli \
   "`dirname "$0"`"/elasticsearch-cli \
   "$@"

distribution/src/bin/elasticsearch-keystore.bat

@@ -4,6 +4,7 @@ setlocal enabledelayedexpansion
 setlocal enableextensions
 
 set ES_MAIN_CLASS=org.elasticsearch.common.settings.KeyStoreCli
+set ES_ADDITIONAL_CLASSPATH_DIRECTORIES=lib/tools/keystore-cli
 call "%~dp0elasticsearch-cli.bat" ^
   %%* ^
   || goto exit

distribution/tools/keystore-cli/build.gradle

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.build'
+
+dependencies {
+  compileOnly "org.elasticsearch:elasticsearch:${version}"
+  compileOnly "org.elasticsearch:elasticsearch-cli:${version}"
+  testCompile "org.elasticsearch.test:framework:${version}"
+  testCompile 'com.google.jimfs:jimfs:1.1'
+  testCompile 'com.google.guava:guava:18.0'
+}

server/build.gradle

@@ -132,8 +132,6 @@ dependencies {
       exclude group: 'org.elasticsearch', module: 'elasticsearch'
     }
   }
-  testCompile 'com.google.jimfs:jimfs:1.1'
-  testCompile 'com.google.guava:guava:18.0'
 }
 
 if (isEclipse) {

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataIndexStateServiceTests.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import com.google.common.collect.ImmutableList;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.ClusterName;
@@ -375,7 +374,8 @@ private static ClusterState addRestoredIndex(final String index, final int numSh
 
         final Snapshot snapshot = new Snapshot(randomAlphaOfLength(10), new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5)));
         final RestoreInProgress.Entry entry =
-            new RestoreInProgress.Entry("_uuid", snapshot, RestoreInProgress.State.INIT, ImmutableList.of(index), shardsBuilder.build());
+            new RestoreInProgress.Entry("_uuid", snapshot, RestoreInProgress.State.INIT,
+                Collections.singletonList(index), shardsBuilder.build());
         return ClusterState.builder(newState)
             .putCustom(RestoreInProgress.TYPE, new RestoreInProgress.Builder().add(entry).build())
             .build();

server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.index.mapper;
 
-import com.google.common.collect.ImmutableMap;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.mapper.FieldMapper.CopyTo;
@@ -27,6 +26,8 @@
 import org.elasticsearch.test.ESTestCase;
 import org.junit.AfterClass;
 
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
@@ -38,11 +39,7 @@ public class ObjectMapperMergeTests extends ESTestCase {
     private static FieldMapper barFieldMapper = createTextFieldMapper("bar");
     private static FieldMapper bazFieldMapper = createTextFieldMapper("baz");
 
-    private static RootObjectMapper rootObjectMapper = createRootObjectMapper(
-        "type1", true, ImmutableMap.of(
-            "disabled", createObjectMapper("disabled", false, emptyMap()),
-            "foo", createObjectMapper("foo", true, ImmutableMap.of(
-                "bar", barFieldMapper))));
+    private static RootObjectMapper rootObjectMapper = createMapping(false, true, true, false);
 
     @AfterClass
     public static void cleanupReferences() {
@@ -51,14 +48,24 @@ public static void cleanupReferences() {
         rootObjectMapper = null;
     }
 
+    private static RootObjectMapper createMapping(boolean disabledFieldEnabled, boolean fooFieldEnabled,
+                                                  boolean includeBarField, boolean includeBazField) {
+        Map<String, Mapper> mappers = new HashMap<>();
+        mappers.put("disabled", createObjectMapper("disabled", disabledFieldEnabled, emptyMap()));
+        Map<String, Mapper> fooMappers = new HashMap<>();
+        if (includeBarField) {
+            fooMappers.put("bar", barFieldMapper);
+        }
+        if (includeBazField) {
+            fooMappers.put("baz", bazFieldMapper);
+        }
+        mappers.put("foo", createObjectMapper("foo", fooFieldEnabled,  Collections.unmodifiableMap(fooMappers)));
+        return createRootObjectMapper("type1", true, Collections.unmodifiableMap(mappers));
+    }
+
     public void testMerge() {
         // GIVEN an enriched mapping with "baz" new field
-        ObjectMapper mergeWith = createRootObjectMapper(
-            "type1", true, ImmutableMap.of(
-                "disabled", createObjectMapper("disabled", false, emptyMap()),
-                "foo", createObjectMapper("foo", true, ImmutableMap.of(
-                    "bar", barFieldMapper,
-                    "baz", bazFieldMapper))));
+        ObjectMapper mergeWith = createMapping(false, true, true, true);
 
         // WHEN merging mappings
         final ObjectMapper merged = rootObjectMapper.merge(mergeWith);
@@ -71,10 +78,7 @@ public void testMerge() {
 
     public void testMergeWhenDisablingField() {
         // GIVEN a mapping with "foo" field disabled
-        ObjectMapper mergeWith = createRootObjectMapper(
-            "type1", true, ImmutableMap.of(
-                "disabled", createObjectMapper("disabled", false, emptyMap()),
-                "foo", createObjectMapper("foo", false, emptyMap())));
+        ObjectMapper mergeWith = createMapping(false, false, false, false);
 
         // WHEN merging mappings
         // THEN a MapperException is thrown with an excepted message
@@ -84,11 +88,7 @@ public void testMergeWhenDisablingField() {
 
     public void testMergeWhenEnablingField() {
         // GIVEN a mapping with "disabled" field enabled
-        ObjectMapper mergeWith = createRootObjectMapper(
-            "type1", true, ImmutableMap.of(
-                "disabled", createObjectMapper("disabled", true, emptyMap()),
-                "foo", createObjectMapper("foo", true, ImmutableMap.of(
-                    "bar", barFieldMapper))));
+        ObjectMapper mergeWith = createMapping(true, true, true, false);
 
         // WHEN merging mappings
         // THEN a MapperException is thrown with an excepted message