CORE: Allow Setting Index Autocreate to _template

* Enables setting autocreate by template as described in elastic#20640
* Closes elastic#20640

docs/reference/docs/index_.asciidoc

@@ -79,6 +79,10 @@ Automatic index creation can include a pattern based white/black list,
 for example, set `action.auto_create_index` to `+aaa*,-bbb*,+ccc*,-*` (+
 meaning allowed, and - meaning disallowed).
 
+Note: The behaviour of `action.auto_create_index` is will change in future versions.
+You can switch to the new template based behaviour by setting its value to `_template`.
+(TODO: link to template setting doc)
+
 [float]
 [[index-versioning]]
 === Versioning

server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequest.java

@@ -76,6 +76,8 @@ public class PutIndexTemplateRequest extends MasterNodeRequest<PutIndexTemplateR
 
     private String cause = "";
 
+    private boolean autoCreateIndex;
+
     private List<String> indexPatterns;
 
     private int order;
@@ -138,6 +140,15 @@ public List<String> patterns() {
         return this.indexPatterns;
     }
 
+    public PutIndexTemplateRequest autoCreateIndex(boolean autoCreateIndex) {
+        this.autoCreateIndex = autoCreateIndex;
+        return this;
+    }
+
+    public boolean autoCreateIndex() {
+        return autoCreateIndex;
+    }
+
     public PutIndexTemplateRequest order(int order) {
         this.order = order;
         return this;
@@ -328,6 +339,8 @@ public PutIndexTemplateRequest source(Map<String, Object> templateSource) {
                 } else {
                     throw new IllegalArgumentException("Malformed [template] value, should be a string or a list of strings");
                 }
+            } else if (name.equals("auto_create_index")) {
+                autoCreateIndex(XContentMapValues.nodeBooleanValue(entry.getValue(), false));
             } else if (name.equals("order")) {
                 order(XContentMapValues.nodeIntegerValue(entry.getValue(), order()));
             } else if ("version".equals(name)) {
@@ -510,6 +523,9 @@ public void readFrom(StreamInput in) throws IOException {
             aliases.add(Alias.read(in));
         }
         version = in.readOptionalVInt();
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            autoCreateIndex = in.readBoolean();
+        }
     }
 
     @Override
@@ -540,11 +556,15 @@ public void writeTo(StreamOutput out) throws IOException {
             alias.writeTo(out);
         }
         out.writeOptionalVInt(version);
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeBoolean(autoCreateIndex);
+        }
     }
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.field("index_patterns", indexPatterns);
+        builder.field("auto_create_index", autoCreateIndex);
         builder.field("order", order);
         if (version != null) {
             builder.field("version", version);

server/src/main/java/org/elasticsearch/action/admin/indices/template/put/PutIndexTemplateRequestBuilder.java

@@ -60,6 +60,11 @@ public PutIndexTemplateRequestBuilder setPatterns(List<String> indexPatterns) {
         return this;
     }
 
+    public PutIndexTemplateRequestBuilder setAutoCreateIndex(boolean autoCreateIndex) {
+        request.autoCreateIndex(autoCreateIndex);
+        return this;
+    }
+
     /**
      * Sets the order of this template if more than one template matches.
      */

server/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java

@@ -80,6 +80,7 @@ protected void masterOperation(final PutIndexTemplateRequest request, final Clus
         indexScopedSettings.validate(templateSettingsBuilder.build(), true); // templates must be consistent with regards to dependencies
         indexTemplateService.putTemplate(new MetaDataIndexTemplateService.PutRequest(cause, request.name())
                 .patterns(request.patterns())
+                .autoCreateIndex(request.autoCreateIndex())
                 .order(request.order())
                 .settings(templateSettingsBuilder.build())
                 .mappings(request.mappings())

server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java

@@ -21,6 +21,7 @@
 
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Tuple;
@@ -41,6 +42,8 @@
  */
 public final class AutoCreateIndex {
 
+    public static final String AUTO_CREATE_TEMPLATE_BASED = "_template";
+
     public static final Setting<AutoCreate> AUTO_CREATE_INDEX_SETTING =
         new Setting<>("action.auto_create_index", "true", AutoCreate::new, Property.NodeScope, Setting.Property.Dynamic);
 
@@ -62,6 +65,10 @@ public boolean needToCheck() {
         return this.autoCreate.autoCreateIndex;
     }
 
+    public boolean templateBased() {
+        return autoCreate.templateBased;
+    }
+
     /**
      * Should the index be auto created?
      * @throws IndexNotFoundException if the index doesn't exist and shouldn't be auto created
@@ -70,6 +77,24 @@ public boolean shouldAutoCreate(String index, ClusterState state) {
         if (resolver.hasIndexOrAlias(index, state)) {
             return false;
         }
+        if (templateBased()) {
+            return MetaDataIndexTemplateService.findTemplates(state.metaData(), index).stream().findFirst()
+                .map(template -> {
+                    if (template.autoCreateIndex()) {
+                        return true;
+                    }
+                    throw new IndexNotFoundException(
+                        "no such index and [" + AUTO_CREATE_INDEX_SETTING.getKey()
+                            + "] is set to [_template] but the matching template does not allow for auto creating the index.",
+                        index
+                    );
+                }).orElseThrow(
+                    () -> new IndexNotFoundException(
+                        "no such index and [" + AUTO_CREATE_INDEX_SETTING.getKey()
+                            + "] is set to [_template] but there is no matching template.", index
+                    )
+                );
+        }
         // One volatile read, so that all checks are done against the same instance:
         final AutoCreate autoCreate = this.autoCreate;
         if (autoCreate.autoCreateIndex == false) {
@@ -108,45 +133,58 @@ void setAutoCreate(AutoCreate autoCreate) {
 
     static class AutoCreate {
         private final boolean autoCreateIndex;
+        private final boolean templateBased;
         private final List<Tuple<String, Boolean>> expressions;
         private final String string;
 
         private AutoCreate(String value) {
             boolean autoCreateIndex;
             List<Tuple<String, Boolean>> expressions = new ArrayList<>();
-            try {
-                autoCreateIndex = Booleans.parseBoolean(value);
-            } catch (IllegalArgumentException ex) {
+            if (AUTO_CREATE_TEMPLATE_BASED.equals(value)) {
+                autoCreateIndex = true;
+                templateBased = true;
+            } else {
+                templateBased = false;
                 try {
-                    String[] patterns = Strings.commaDelimitedListToStringArray(value);
-                    for (String pattern : patterns) {
-                        if (pattern == null || pattern.trim().length() == 0) {
-                            throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] must "
-                                    + "be either [true, false, or a comma separated list of index patterns]");
-                        }
-                        pattern = pattern.trim();
-                        Tuple<String, Boolean> expression;
-                        if (pattern.startsWith("-")) {
-                            if (pattern.length() == 1) {
-                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] "
-                                        + "must contain an index name after [-]");
+                    autoCreateIndex = Booleans.parseBoolean(value);
+                } catch (IllegalArgumentException ex) {
+                    try {
+                        String[] patterns = Strings.commaDelimitedListToStringArray(value);
+                        for (String pattern : patterns) {
+                            if (pattern == null || pattern.trim().length() == 0) {
+                                throw new IllegalArgumentException(
+                                    "Can't parse [" + value + "] for setting [action.auto_create_index] must "
+                                    + "be either [true, false, or a comma separated list of index patterns]"
+                                );
                             }
-                            expression = new Tuple<>(pattern.substring(1), false);
-                        } else if(pattern.startsWith("+")) {
-                            if (pattern.length() == 1) {
-                                throw new IllegalArgumentException("Can't parse [" + value + "] for setting [action.auto_create_index] "
-                                        + "must contain an index name after [+]");
+                            pattern = pattern.trim();
+                            Tuple<String, Boolean> expression;
+                            if (pattern.startsWith("-")) {
+                                if (pattern.length() == 1) {
+                                    throw new IllegalArgumentException(
+                                        "Can't parse [" + value + "] for setting [action.auto_create_index] "
+                                        + "must contain an index name after [-]"
+                                    );
+                                }
+                                expression = new Tuple<>(pattern.substring(1), false);
+                            } else if (pattern.startsWith("+")) {
+                                if (pattern.length() == 1) {
+                                    throw new IllegalArgumentException(
+                                        "Can't parse [" + value + "] for setting [action.auto_create_index] "
+                                        + "must contain an index name after [+]"
+                                    );
+                                }
+                                expression = new Tuple<>(pattern.substring(1), true);
+                            } else {
+                                expression = new Tuple<>(pattern, true);
                             }
-                            expression = new Tuple<>(pattern.substring(1), true);
-                        } else {
-                            expression = new Tuple<>(pattern, true);
+                            expressions.add(expression);
                         }
-                        expressions.add(expression);
+                        autoCreateIndex = true;
+                    } catch (IllegalArgumentException ex1) {
+                        ex1.addSuppressed(ex);
+                        throw ex1;
                     }
-                    autoCreateIndex = true;
-                } catch (IllegalArgumentException ex1) {
-                    ex1.addSuppressed(ex);
-                    throw ex1;
                 }
             }
             this.expressions = expressions;

server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java

@@ -80,6 +80,8 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
 
     private final List<String> patterns;
 
+    private final boolean autoCreateIndex;
+
     private final Settings settings;
 
     // the mapping source should always include the type as top level
@@ -93,14 +95,16 @@ public IndexTemplateMetaData(String name, int order, Integer version,
                                  List<String> patterns, Settings settings,
                                  ImmutableOpenMap<String, CompressedXContent> mappings,
                                  ImmutableOpenMap<String, AliasMetaData> aliases,
-                                 ImmutableOpenMap<String, IndexMetaData.Custom> customs) {
+                                 ImmutableOpenMap<String, IndexMetaData.Custom> customs,
+                                 boolean autoCreateIndex) {
         if (patterns == null || patterns.isEmpty()) {
             throw new IllegalArgumentException("Index patterns must not be null or empty; got " + patterns);
         }
         this.name = name;
         this.order = order;
         this.version = version;
         this.patterns= patterns;
+        this.autoCreateIndex = autoCreateIndex;
         this.settings = settings;
         this.mappings = mappings;
         this.aliases = aliases;
@@ -141,6 +145,14 @@ public List<String> getPatterns() {
         return this.patterns;
     }
 
+    public boolean getAutoCreateIndex() {
+        return this.autoCreateIndex;
+    }
+
+    public boolean autoCreateIndex() {
+        return this.autoCreateIndex;
+    }
+
     public Settings settings() {
         return this.settings;
     }
@@ -234,6 +246,9 @@ public static IndexTemplateMetaData readFrom(StreamInput in) throws IOException
             builder.putCustom(type, customIndexMetaData);
         }
         builder.version(in.readOptionalVInt());
+        if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            builder.autoCreateIndex(in.readBoolean());
+        }
         return builder.build();
     }
 
@@ -266,12 +281,15 @@ public void writeTo(StreamOutput out) throws IOException {
             cursor.value.writeTo(out);
         }
         out.writeOptionalVInt(version);
+        if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
+            out.writeBoolean(autoCreateIndex);
+        }
     }
 
     public static class Builder {
 
         private static final Set<String> VALID_FIELDS = Sets.newHashSet(
-            "template", "order", "mappings", "settings", "index_patterns", "aliases", "version");
+            "template", "order", "mappings", "settings", "index_patterns", "auto_create_index" , "aliases", "version");
         static {
             VALID_FIELDS.addAll(IndexMetaData.customPrototypes.keySet());
         }
@@ -284,6 +302,8 @@ public static class Builder {
 
         private List<String> indexPatterns;
 
+        private boolean autoCreateIndex;
+
         private Settings settings = Settings.Builder.EMPTY_SETTINGS;
 
         private final ImmutableOpenMap.Builder<String, CompressedXContent> mappings;
@@ -326,6 +346,10 @@ public Builder patterns(List<String> indexPatterns) {
             return this;
         }
 
+        public Builder autoCreateIndex(boolean autoCreateIndex) {
+            this.autoCreateIndex = autoCreateIndex;
+            return this;
+        }
 
         public Builder settings(Settings.Builder settings) {
             this.settings = settings.build();
@@ -378,7 +402,7 @@ public IndexMetaData.Custom getCustom(String type) {
 
         public IndexTemplateMetaData build() {
             return new IndexTemplateMetaData(name, order, version, indexPatterns, settings, mappings.build(),
-                aliases.build(), customs.build());
+                aliases.build(), customs.build(), autoCreateIndex);
         }
 
         public static void toXContent(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder, ToXContent.Params params)
@@ -398,6 +422,7 @@ public static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData,
                 builder.field("version", indexTemplateMetaData.version());
             }
             builder.field("index_patterns", indexTemplateMetaData.patterns());
+            builder.field("auto_create_index", indexTemplateMetaData.autoCreateIndex());
 
             builder.startObject("settings");
             indexTemplateMetaData.settings().toXContent(builder, params);
@@ -509,6 +534,8 @@ public static IndexTemplateMetaData fromXContent(XContentParser parser, String t
                         builder.order(parser.intValue());
                     } else if ("version".equals(currentFieldName)) {
                         builder.version(parser.intValue());
+                    } else if ("auto_create_index".equals(currentFieldName)) {
+                        builder.autoCreateIndex(parser.booleanValue());
                     }
                 }
             }

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java

@@ -240,6 +240,7 @@ private static void validateAndAddTemplate(final PutRequest request, IndexTempla
             templateBuilder.order(request.order);
             templateBuilder.version(request.version);
             templateBuilder.patterns(request.indexPatterns);
+            templateBuilder.autoCreateIndex(request.autoCreateIndex);
             templateBuilder.settings(request.settings);
 
             Map<String, Map<String, Object>> mappingsForValidation = new HashMap<>();
@@ -336,6 +337,7 @@ public static class PutRequest {
         int order;
         Integer version;
         List<String> indexPatterns;
+        boolean autoCreateIndex;
         Settings settings = Settings.Builder.EMPTY_SETTINGS;
         Map<String, String> mappings = new HashMap<>();
         List<Alias> aliases = new ArrayList<>();
@@ -358,6 +360,11 @@ public PutRequest patterns(List<String> indexPatterns) {
             return this;
         }
 
+        public PutRequest autoCreateIndex(boolean autoCreateIndex) {
+            this.autoCreateIndex = autoCreateIndex;
+            return this;
+        }
+
         public PutRequest create(boolean create) {
             this.create = create;
             return this;

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java

@@ -59,6 +59,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             putRequest.patterns(Arrays.asList(request.paramAsStringArray("index_patterns", Strings.EMPTY_ARRAY)));
         }
         putRequest.order(request.paramAsInt("order", putRequest.order()));
+        putRequest.autoCreateIndex(request.paramAsBoolean("auto_create_index", false));
         putRequest.masterNodeTimeout(request.paramAsTime("master_timeout", putRequest.masterNodeTimeout()));
         putRequest.create(request.paramAsBoolean("create", false));
         putRequest.cause(request.param("cause", ""));