Merge branch 'master' into feature/client_aggs_parsing

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -167,6 +167,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
                  * warning every time. */
                 current.println("  - skip:")
                 current.println("      features: ")
+                current.println("        - stash_in_key")
                 current.println("        - warnings")
             }
             if (test.skipTest) {
@@ -295,7 +296,7 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
             Path dest = outputRoot().toPath().resolve(test.path)
             // Replace the extension
             String fileName = dest.getName(dest.nameCount - 1)
-            dest = dest.parent.resolve(fileName.replace('.asciidoc', '.yaml'))
+            dest = dest.parent.resolve(fileName.replace('.asciidoc', '.yml'))
 
             // Now setup the writer
             Files.createDirectories(dest.parent)

core/src/main/java/org/elasticsearch/ElasticsearchException.java

@@ -133,7 +133,7 @@ public ElasticsearchException(StreamInput in) throws IOException {
         super(in.readOptionalString(), in.readException());
         readStackTrace(this, in);
         headers.putAll(in.readMapOfLists(StreamInput::readString, StreamInput::readString));
-        if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
             metadata.putAll(in.readMapOfLists(StreamInput::readString, StreamInput::readString));
         } else {
             for (Iterator<Map.Entry<String, List<String>>> iterator = headers.entrySet().iterator(); iterator.hasNext(); ) {
@@ -284,7 +284,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(this.getMessage());
         out.writeException(this.getCause());
         writeStackTraces(this, out);
-        if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
             out.writeMapOfLists(headers, StreamOutput::writeString, StreamOutput::writeString);
             out.writeMapOfLists(metadata, StreamOutput::writeString, StreamOutput::writeString);
         } else {
@@ -985,11 +985,11 @@ private enum ElasticsearchExceptionHandle {
         STATUS_EXCEPTION(org.elasticsearch.ElasticsearchStatusException.class, org.elasticsearch.ElasticsearchStatusException::new, 145,
             UNKNOWN_VERSION_ADDED),
         TASK_CANCELLED_EXCEPTION(org.elasticsearch.tasks.TaskCancelledException.class,
-            org.elasticsearch.tasks.TaskCancelledException::new, 146, Version.V_5_1_1_UNRELEASED),
+            org.elasticsearch.tasks.TaskCancelledException::new, 146, Version.V_5_1_1),
         SHARD_LOCK_OBTAIN_FAILED_EXCEPTION(org.elasticsearch.env.ShardLockObtainFailedException.class,
                                            org.elasticsearch.env.ShardLockObtainFailedException::new, 147, Version.V_5_0_2),
         UNKNOWN_NAMED_OBJECT_EXCEPTION(org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException.class,
-                org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException::new, 148, Version.V_5_2_0_UNRELEASED);
+                org.elasticsearch.common.xcontent.NamedXContentRegistry.UnknownNamedObjectException::new, 148, Version.V_5_2_0);
 
         final Class<? extends ElasticsearchException> exceptionClass;
         final CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> constructor;

core/src/main/java/org/elasticsearch/Version.java

@@ -55,33 +55,25 @@ public class Version implements Comparable<Version> {
     public static final Version V_5_0_1 = new Version(V_5_0_1_ID, org.apache.lucene.util.Version.LUCENE_6_2_1);
     public static final int V_5_0_2_ID = 5000299;
     public static final Version V_5_0_2 = new Version(V_5_0_2_ID, org.apache.lucene.util.Version.LUCENE_6_2_1);
-    public static final int V_5_0_3_ID_UNRELEASED = 5000399;
-    public static final Version V_5_0_3_UNRELEASED = new Version(V_5_0_3_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
     // no version constant for 5.1.0 due to inadvertent release
-    public static final int V_5_1_1_ID_UNRELEASED = 5010199;
-    public static final Version V_5_1_1_UNRELEASED = new Version(V_5_1_1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
-    public static final int V_5_1_2_ID_UNRELEASED = 5010299;
-    public static final Version V_5_1_2_UNRELEASED = new Version(V_5_1_2_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
-    public static final int V_5_1_3_ID_UNRELEASED = 5010399;
-    public static final Version V_5_1_3_UNRELEASED = new Version(V_5_1_3_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_3_0);
-    public static final int V_5_2_0_ID_UNRELEASED = 5020099;
-    public static final Version V_5_2_0_UNRELEASED = new Version(V_5_2_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_0);
-    public static final int V_5_2_1_ID_UNRELEASED = 5020199;
-    public static final Version V_5_2_1_UNRELEASED = new Version(V_5_2_1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_1);
-    public static final int V_5_2_2_ID_UNRELEASED = 5020299;
-    public static final Version V_5_2_2_UNRELEASED = new Version(V_5_2_2_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_1);
-    public static final int V_5_2_3_ID_UNRELEASED = 5020399;
-    public static final Version V_5_2_3_UNRELEASED = new Version(V_5_2_3_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_1);
-    public static final int V_5_3_0_ID_UNRELEASED = 5030099;
-    public static final Version V_5_3_0_UNRELEASED = new Version(V_5_3_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_1);
-    public static final int V_5_3_1_ID_UNRELEASED = 5030199;
-    public static final Version V_5_3_1_UNRELEASED = new Version(V_5_3_1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_2);
-    public static final int V_5_3_2_ID_UNRELEASED = 5030299;
-    public static final Version V_5_3_2_UNRELEASED = new Version(V_5_3_2_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_4_2);
-    public static final int V_5_4_0_ID_UNRELEASED = 5040099;
-    public static final Version V_5_4_0_UNRELEASED = new Version(V_5_4_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_5_0);
-    public static final int V_5_4_1_ID_UNRELEASED = 5040199;
-    public static final Version V_5_4_1_UNRELEASED = new Version(V_5_4_1_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_5_1);
+    public static final int V_5_1_1_ID = 5010199;
+    public static final Version V_5_1_1 = new Version(V_5_1_1_ID, org.apache.lucene.util.Version.LUCENE_6_3_0);
+    public static final int V_5_1_2_ID = 5010299;
+    public static final Version V_5_1_2 = new Version(V_5_1_2_ID, org.apache.lucene.util.Version.LUCENE_6_3_0);
+    public static final int V_5_2_0_ID = 5020099;
+    public static final Version V_5_2_0 = new Version(V_5_2_0_ID, org.apache.lucene.util.Version.LUCENE_6_4_0);
+    public static final int V_5_2_1_ID = 5020199;
+    public static final Version V_5_2_1 = new Version(V_5_2_1_ID, org.apache.lucene.util.Version.LUCENE_6_4_1);
+    public static final int V_5_2_2_ID = 5020299;
+    public static final Version V_5_2_2 = new Version(V_5_2_2_ID, org.apache.lucene.util.Version.LUCENE_6_4_1);
+    public static final int V_5_3_0_ID = 5030099;
+    public static final Version V_5_3_0 = new Version(V_5_3_0_ID, org.apache.lucene.util.Version.LUCENE_6_4_1);
+    public static final int V_5_3_1_ID = 5030199;
+    public static final Version V_5_3_1 = new Version(V_5_3_1_ID, org.apache.lucene.util.Version.LUCENE_6_4_2);
+    public static final int V_5_3_2_ID = 5030299;
+    public static final Version V_5_3_2 = new Version(V_5_3_2_ID, org.apache.lucene.util.Version.LUCENE_6_4_2);
+    public static final int V_5_4_0_ID = 5040099;
+    public static final Version V_5_4_0 = new Version(V_5_4_0_ID, org.apache.lucene.util.Version.LUCENE_6_5_0);
     public static final int V_5_5_0_ID_UNRELEASED = 5050099;
     public static final Version V_5_5_0_UNRELEASED = new Version(V_5_5_0_ID_UNRELEASED, org.apache.lucene.util.Version.LUCENE_6_5_0);
     public static final int V_6_0_0_alpha1_ID_UNRELEASED = 6000001;
@@ -111,32 +103,24 @@ public static Version fromId(int id) {
                 return V_6_0_0_alpha1_UNRELEASED;
             case V_5_5_0_ID_UNRELEASED:
                 return V_5_5_0_UNRELEASED;
-            case V_5_4_1_ID_UNRELEASED:
-                return V_5_4_1_UNRELEASED;
-            case V_5_4_0_ID_UNRELEASED:
-                return V_5_4_0_UNRELEASED;
-            case V_5_3_2_ID_UNRELEASED:
-                return V_5_3_2_UNRELEASED;
-            case V_5_3_1_ID_UNRELEASED:
-                return V_5_3_1_UNRELEASED;
-            case V_5_3_0_ID_UNRELEASED:
-                return V_5_3_0_UNRELEASED;
-            case V_5_2_3_ID_UNRELEASED:
-                return V_5_2_3_UNRELEASED;
-            case V_5_2_2_ID_UNRELEASED:
-                return V_5_2_2_UNRELEASED;
-            case V_5_2_1_ID_UNRELEASED:
-                return V_5_2_1_UNRELEASED;
-            case V_5_2_0_ID_UNRELEASED:
-                return V_5_2_0_UNRELEASED;
-            case V_5_1_3_ID_UNRELEASED:
-                return V_5_1_3_UNRELEASED;
-            case V_5_1_2_ID_UNRELEASED:
-                return V_5_1_2_UNRELEASED;
-            case V_5_1_1_ID_UNRELEASED:
-                return V_5_1_1_UNRELEASED;
-            case V_5_0_3_ID_UNRELEASED:
-                return V_5_0_3_UNRELEASED;
+            case V_5_4_0_ID:
+                return V_5_4_0;
+            case V_5_3_2_ID:
+                return V_5_3_2;
+            case V_5_3_1_ID:
+                return V_5_3_1;
+            case V_5_3_0_ID:
+                return V_5_3_0;
+            case V_5_2_2_ID:
+                return V_5_2_2;
+            case V_5_2_1_ID:
+                return V_5_2_1;
+            case V_5_2_0_ID:
+                return V_5_2_0;
+            case V_5_1_2_ID:
+                return V_5_1_2;
+            case V_5_1_1_ID:
+                return V_5_1_1;
             case V_5_0_2_ID:
                 return V_5_0_2;
             case V_5_0_1_ID:
@@ -296,7 +280,7 @@ public Version minimumCompatibilityVersion() {
         final int bwcMinor;
         if (this.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) {
             bwcMajor = major - 1;
-            bwcMinor = 0; // TODO we have to move this to the latest released minor of the last major but for now we just keep
+            bwcMinor = 4;
         } else {
             bwcMajor = major;
             bwcMinor = 0;
@@ -306,7 +290,8 @@ public Version minimumCompatibilityVersion() {
 
     /**
      * Returns the minimum created index version that this version supports. Indices created with lower versions
-     * can't be used with this version.
+     * can't be used with this version. This should also be used for file based serialization backwards compatibility ie. on serialization
+     * code that is used to read / write file formats like transaction logs, cluster state, and index metadata.
      */
     public Version minimumIndexCompatibilityVersion() {
         final int bwcMajor;

core/src/main/java/org/elasticsearch/action/OriginalIndices.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
+import java.util.Arrays;
 
 /**
  * Used to keep track of original indices within internal (e.g. shard level) requests
@@ -64,4 +65,12 @@ public static void writeOriginalIndices(OriginalIndices originalIndices, StreamO
         out.writeStringArrayNullable(originalIndices.indices);
         originalIndices.indicesOptions.writeIndicesOptions(out);
     }
+
+    @Override
+    public String toString() {
+        return "OriginalIndices{" +
+            "indices=" + Arrays.toString(indices) +
+            ", indicesOptions=" + indicesOptions +
+            '}';
+    }
 }

core/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainRequest.java

@@ -249,8 +249,8 @@ public void writeTo(StreamOutput out) throws IOException {
     }
 
     private void checkVersion(Version version) {
-        if (version.before(Version.V_5_2_0_UNRELEASED)) {
-            throw new IllegalArgumentException("cannot explain shards in a mixed-cluster with pre-" + Version.V_5_2_0_UNRELEASED +
+        if (version.before(Version.V_5_2_0)) {
+            throw new IllegalArgumentException("cannot explain shards in a mixed-cluster with pre-" + Version.V_5_2_0 +
                                                " nodes, node version [" + version + "]");
         }
     }

core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java

@@ -67,12 +67,20 @@ synchronized ClusterState updateSettings(final ClusterState currentState, Settin
             .transientSettings(transientSettings.build());
 
         ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
-        boolean updatedReadOnly = MetaData.SETTING_READ_ONLY_SETTING.get(metaData.persistentSettings()) || MetaData.SETTING_READ_ONLY_SETTING.get(metaData.transientSettings());
+        boolean updatedReadOnly = MetaData.SETTING_READ_ONLY_SETTING.get(metaData.persistentSettings())
+            || MetaData.SETTING_READ_ONLY_SETTING.get(metaData.transientSettings());
         if (updatedReadOnly) {
             blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK);
         } else {
             blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_BLOCK);
         }
+        boolean updatedReadOnlyAllowDelete = MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.get(metaData.persistentSettings())
+            || MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.get(metaData.transientSettings());
+        if (updatedReadOnlyAllowDelete) {
+            blocks.addGlobalBlock(MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK);
+        } else {
+            blocks.removeGlobalBlock(MetaData.CLUSTER_READ_ONLY_ALLOW_DELETE_BLOCK);
+        }
         ClusterState build = builder(currentState).metaData(metaData).blocks(blocks).build();
         Settings settings = build.metaData().settings();
         // now we try to apply things and if they are invalid we fail

core/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java

@@ -67,12 +67,15 @@ protected String executor() {
     @Override
     protected ClusterBlockException checkBlock(ClusterUpdateSettingsRequest request, ClusterState state) {
         // allow for dedicated changes to the metadata blocks, so we don't block those to allow to "re-enable" it
-        if ((request.transientSettings().isEmpty() &&
-            request.persistentSettings().size() == 1 &&
-            MetaData.SETTING_READ_ONLY_SETTING.exists(request.persistentSettings())) ||
-            (request.persistentSettings().isEmpty() && request.transientSettings().size() == 1 &&
-                MetaData.SETTING_READ_ONLY_SETTING.exists(request.transientSettings()))) {
-            return null;
+        if (request.transientSettings().size() + request.persistentSettings().size() == 1) {
+            // only one setting
+            if (MetaData.SETTING_READ_ONLY_SETTING.exists(request.persistentSettings())
+                || MetaData.SETTING_READ_ONLY_SETTING.exists(request.transientSettings())
+                || MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.exists(request.transientSettings())
+                || MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING.exists(request.persistentSettings())) {
+                // one of the settings above as the only setting in the request means - resetting the block!
+                return null;
+            }
         }
         return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);
     }

core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequest.java

@@ -134,7 +134,7 @@ public void readFrom(StreamInput in) throws IOException {
         routing = in.readOptionalString();
         preference = in.readOptionalString();
 
-        if (in.getVersion().onOrBefore(Version.V_5_1_1_UNRELEASED)) {
+        if (in.getVersion().onOrBefore(Version.V_5_1_1)) {
             //types
             in.readStringArray();
         }
@@ -153,7 +153,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalString(routing);
         out.writeOptionalString(preference);
 
-        if (out.getVersion().onOrBefore(Version.V_5_1_1_UNRELEASED)) {
+        if (out.getVersion().onOrBefore(Version.V_5_1_1)) {
             //types
             out.writeStringArray(Strings.EMPTY_ARRAY);
         }

core/src/main/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsResponse.java

@@ -73,7 +73,7 @@ public void readFrom(StreamInput in) throws IOException {
         for (int i = 0; i < nodes.length; i++) {
             nodes[i] = new DiscoveryNode(in);
         }
-        if (in.getVersion().onOrAfter(Version.V_5_1_1_UNRELEASED)) {
+        if (in.getVersion().onOrAfter(Version.V_5_1_1)) {
             int size = in.readVInt();
             indicesAndFilters = new HashMap<>();
             for (int i = 0; i < size; i++) {
@@ -95,7 +95,7 @@ public void writeTo(StreamOutput out) throws IOException {
         for (DiscoveryNode node : nodes) {
             node.writeTo(out);
         }
-        if (out.getVersion().onOrAfter(Version.V_5_1_1_UNRELEASED)) {
+        if (out.getVersion().onOrAfter(Version.V_5_1_1)) {
             out.writeVInt(indicesAndFilters.size());
             for (Map.Entry<String, AliasFilter> entry : indicesAndFilters.entrySet()) {
                 out.writeString(entry.getKey());

core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptResponse.java

@@ -59,7 +59,7 @@ public void readFrom(StreamInput in) throws IOException {
         super.readFrom(in);
 
         if (in.readBoolean()) {
-            if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+            if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
                 source = new StoredScriptSource(in);
             } else {
                 source = new StoredScriptSource(in.readString());
@@ -78,7 +78,7 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             out.writeBoolean(true);
 
-            if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+            if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
                 source.writeTo(out);
             } else {
                 out.writeString(source.getCode());

core/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java

@@ -123,7 +123,7 @@ public void readFrom(StreamInput in) throws IOException {
 
         id = in.readOptionalString();
         content = in.readBytesReference();
-        if (in.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+        if (in.getVersion().onOrAfter(Version.V_5_3_0)) {
             xContentType = XContentType.readFrom(in);
         } else {
             xContentType = XContentFactory.xContentType(content);
@@ -137,7 +137,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(lang == null ? "" : lang);
         out.writeOptionalString(id);
         out.writeBytesReference(content);
-        if (out.getVersion().onOrAfter(Version.V_5_3_0_UNRELEASED)) {
+        if (out.getVersion().onOrAfter(Version.V_5_3_0)) {
             xContentType.writeTo(out);
         }
     }

core/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeResponse.java

@@ -117,7 +117,7 @@ public void readFrom(StreamInput in) throws IOException {
             startOffset = in.readInt();
             endOffset = in.readInt();
             position = in.readVInt();
-            if (in.getVersion().onOrAfter(Version.V_5_2_0_UNRELEASED)) {
+            if (in.getVersion().onOrAfter(Version.V_5_2_0)) {
                 Integer len = in.readOptionalVInt();
                 if (len != null) {
                     positionLength = len;
@@ -135,7 +135,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeInt(startOffset);
             out.writeInt(endOffset);
             out.writeVInt(position);
-            if (out.getVersion().onOrAfter(Version.V_5_2_0_UNRELEASED)) {
+            if (out.getVersion().onOrAfter(Version.V_5_2_0)) {
                 out.writeOptionalVInt(positionLength > 1 ? positionLength : null);
             }
             out.writeOptionalString(type);

core/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java

@@ -78,7 +78,7 @@ protected void doExecute(Task task, DeleteIndexRequest request, ActionListener<D
 
     @Override
     protected ClusterBlockException checkBlock(DeleteIndexRequest request, ClusterState state) {
-        return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_WRITE, indexNameExpressionResolver.concreteIndexNames(state, request));
+        return state.blocks().indicesAllowReleaseResources(indexNameExpressionResolver.concreteIndexNames(state, request));
     }
 
     @Override