Eager norms loading options.

Norms can be eagerly loaded on a per-field basis by setting norms.loading to
`eager` instead of the default `lazy`:

```
"my_string_field" : {
  "type": "string",
  "norms": {
    "loading": "eager"
  }
}
```

In case this behavior should be applied to all fields, it is possible to change
the default value by setting `index.norms.loading` to `eager`.

Close #4079

docs/reference/mapping/types/core-types.asciidoc

@@ -55,6 +55,13 @@ more characters. An example mapping can be:
                 "store" : "yes",
                 "index" : "analyzed",
                 "null_value" : "na"
+            },
+            "user" : {
+                "type" : "string",
+                "index" : "not_analyzed",
+                "norms" : {
+                    "enabled" : false
+                }
             }
         }
     }
@@ -65,7 +72,10 @@ The above mapping defines a `string` `message` property/field within the
 `tweet` type. The field is stored in the index (so it can later be
 retrieved using selective loading when searching), and it gets analyzed
 (broken down into searchable terms). If the message has a `null` value,
-then the value that will be stored is `na`.
+then the value that will be stored is `na`. There is also a `string` `user`
+which is indexed as-is (not broken down into tokens) and has norms
+disabled (so that matching this field is a binary decision, no match is
+better than another one).
 
 The following table lists all the attributes that can be used with the
 `string` type:
@@ -97,9 +107,13 @@ searchable at all (as an individual field; it may still be included in
 `null_value` as the field value. Defaults to not adding the field at
 all.
 
-|`omit_norms` |Boolean value if norms should be omitted or not. Defaults
-to `false` for `analyzed` fields, and to `true` for `not_analyzed`
-fields.
+|`norms.enabled` |Boolean value if norms should be enabled or not. Defaults
+to `true` for `analyzed` fields, and to `false` for `not_analyzed` fields.
+
+|`norms.loading` |Describes how norms should be loaded, possible values are
+`eager` and `lazy` (default). It is possible to change the default value to
+eager for all fields by configuring the index setting `index.norms.loading`
+to `eager`.
 
 |`omit_term_freq_and_positions` |Boolean value if term freq and
 positions should be omitted. Defaults to `false`. Deprecated since 0.20,

src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java

@@ -21,6 +21,7 @@
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import org.elasticsearch.ElasticSearchParseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.unit.TimeValue;
@@ -383,4 +384,12 @@ public static TimeValue nodeTimeValue(Object node) {
         }
         return TimeValue.parseTimeValue(node.toString(), null);
     }
+
+    public static Map<String, Object> nodeMapValue(Object node, String desc) {
+        if (node instanceof Map) {
+            return (Map<String, Object>) node;
+        } else {
+            throw new ElasticSearchParseException(desc + " should be a hash but was of type: " + node.getClass());
+        }
+    }
 }

src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java

@@ -21,21 +21,12 @@
 
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.index.mapper.MapperParsingException;
+import org.elasticsearch.index.mapper.FieldMapper.Loading;
 
 /**
  */
 public class FieldDataType {
 
-    public static final String FORMAT_KEY = "format";
-    private static final String LOADING_KEY = "loading";
-    private static final String EAGER_LOADING_VALUE = "eager";
-    private static final String LAZY_LOADING_VALUE = "lazy";
-
-    public static enum Loading {
-        LAZY, EAGER;
-    }
-
     private final String type;
     private final Loading loading;
     private final Settings settings;
@@ -51,14 +42,8 @@ public FieldDataType(String type, Settings.Builder builder) {
     public FieldDataType(String type, Settings settings) {
         this.type = type;
         this.settings = settings;
-        final String loading = settings.get(LOADING_KEY);
-        if (loading == null || loading.equals(LAZY_LOADING_VALUE)) {
-            this.loading = Loading.LAZY;
-        } else if (loading.equals(EAGER_LOADING_VALUE)) {
-            this.loading = Loading.EAGER;
-        } else {
-            throw new MapperParsingException("Unknown [" + LOADING_KEY + "] value: [" + loading + "]");
-        }
+        final String loading = settings.get(Loading.KEY);
+        this.loading = Loading.parse(loading, Loading.LAZY);
     }
 
     public String getType() {

src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import com.google.common.base.Strings;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.Term;
@@ -119,6 +120,38 @@ public Term createIndexNameTerm(BytesRef value) {
         }
     }
 
+    public static enum Loading {
+        LAZY {
+            @Override
+            public String toString() {
+                return LAZY_VALUE;
+            }
+        },
+        EAGER {
+            @Override
+            public String toString() {
+                return EAGER_VALUE;
+            }
+        };
+
+        public static final String KEY = "loading";
+        public static final String EAGER_VALUE = "eager";
+        public static final String LAZY_VALUE = "lazy";
+
+        public static Loading parse(String loading, Loading defaultValue) {
+            if (Strings.isNullOrEmpty(loading)) {
+                return defaultValue;
+            } else if (EAGER_VALUE.equalsIgnoreCase(loading)) {
+                return EAGER;
+            } else if (LAZY_VALUE.equalsIgnoreCase(loading)) {
+                return LAZY;
+            } else {
+                throw new MapperParsingException("Unknown [" + KEY + "] value: [" + loading + "]");
+            }
+        }
+
+    }
+
     Names names();
 
     FieldType fieldType();
@@ -207,4 +240,6 @@ public Term createIndexNameTerm(BytesRef value) {
     boolean isNumeric();
 
     boolean isSortable();
+
+    Loading normsLoading(Loading defaultLoading);
 }

src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java

@@ -168,6 +168,7 @@ public abstract static class Builder<T extends Builder, Y extends AbstractFieldM
         protected boolean indexOptionsSet = false;
         protected PostingsFormatProvider provider;
         protected SimilarityProvider similarity;
+        protected Loading normsLoading;
         @Nullable
         protected Settings fieldDataSettings;
 
@@ -269,7 +270,12 @@ protected T similarity(SimilarityProvider similarity) {
             return builder;
         }
 
-        protected T fieldDataSettings(Settings settings) {
+        public T normsLoading(Loading normsLoading) {
+            this.normsLoading = normsLoading;
+            return builder;
+        }
+
+        public T fieldDataSettings(Settings settings) {
             this.fieldDataSettings = settings;
             return builder;
         }
@@ -295,13 +301,13 @@ protected String buildFullName(BuilderContext context) {
     protected NamedAnalyzer searchAnalyzer;
     protected PostingsFormatProvider postingsFormat;
     protected final SimilarityProvider similarity;
-
+    protected Loading normsLoading;
     protected Settings customFieldDataSettings;
     protected FieldDataType fieldDataType;
 
     protected AbstractFieldMapper(Names names, float boost, FieldType fieldType, NamedAnalyzer indexAnalyzer,
                                   NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsFormat, SimilarityProvider similarity,
-                                  @Nullable Settings fieldDataSettings) {
+                                  Loading normsLoading, @Nullable Settings fieldDataSettings) {
         this.names = names;
         this.boost = boost;
         this.fieldType = fieldType;
@@ -326,6 +332,7 @@ protected AbstractFieldMapper(Names names, float boost, FieldType fieldType, Nam
         }
         this.postingsFormat = postingsFormat;
         this.similarity = similarity;
+        this.normsLoading = normsLoading;
 
         this.customFieldDataSettings = fieldDataSettings;
         if (fieldDataSettings == null) {
@@ -594,6 +601,7 @@ public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappi
         if (!mergeContext.mergeFlags().simulate()) {
             // apply changeable values
             this.boost = fieldMergeWith.boost;
+            this.normsLoading = fieldMergeWith.normsLoading;
             if (fieldMergeWith.postingsFormat != null) {
                 this.postingsFormat = fieldMergeWith.postingsFormat;
             }
@@ -646,8 +654,15 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
         if (includeDefaults || fieldType.storeTermVectors() != defaultFieldType.storeTermVectors()) {
             builder.field("term_vector", termVectorOptionsToString(fieldType));
         }
-        if (includeDefaults || fieldType.omitNorms() != defaultFieldType.omitNorms()) {
-            builder.field("omit_norms", fieldType.omitNorms());
+        if (includeDefaults || fieldType.omitNorms() != defaultFieldType.omitNorms() || normsLoading != null) {
+            builder.startObject("norms");
+            if (includeDefaults || fieldType.omitNorms() != defaultFieldType.omitNorms()) {
+                builder.field("enabled", !fieldType.omitNorms());
+            }
+            if (normsLoading != null) {
+                builder.field(Loading.KEY, normsLoading);
+            }
+            builder.endObject();
         }
         if (includeDefaults || fieldType.indexOptions() != defaultFieldType.indexOptions()) {
             builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
@@ -771,4 +786,9 @@ public boolean isNumeric() {
     public boolean isSortable() {
         return true;
     }
+
+    @Override
+    public Loading normsLoading(Loading defaultLoading) {
+        return normsLoading == null ? defaultLoading : normsLoading;
+    }
 }

src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java

@@ -123,7 +123,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
     private long compressThreshold;
 
     protected BinaryFieldMapper(Names names, FieldType fieldType, Boolean compress, long compressThreshold, PostingsFormatProvider provider) {
-        super(names, 1.0f, fieldType, null, null, provider, null, null);
+        super(names, 1.0f, fieldType, null, null, provider, null, null, null);
         this.compress = compress;
         this.compressThreshold = compressThreshold;
     }
@@ -235,4 +235,4 @@ public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappi
         }
     }
 
-}
+}

src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java

@@ -97,7 +97,7 @@ protected Builder tokenized(boolean tokenized) {
 
         @Override
         public BooleanFieldMapper build(BuilderContext context) {
-            return new BooleanFieldMapper(buildNames(context), boost, fieldType, nullValue, provider, similarity, fieldDataSettings);
+            return new BooleanFieldMapper(buildNames(context), boost, fieldType, nullValue, provider, similarity, normsLoading, fieldDataSettings);
         }
     }
 
@@ -119,8 +119,8 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
 
     private Boolean nullValue;
 
-    protected BooleanFieldMapper(Names names, float boost, FieldType fieldType, Boolean nullValue, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
-        super(names, boost, fieldType, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, provider, similarity, fieldDataSettings);
+    protected BooleanFieldMapper(Names names, float boost, FieldType fieldType, Boolean nullValue, PostingsFormatProvider provider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings) {
+        super(names, boost, fieldType, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, provider, similarity, normsLoading, fieldDataSettings);
         this.nullValue = nullValue;
     }
 

src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java

@@ -90,7 +90,7 @@ public ByteFieldMapper build(BuilderContext context) {
             fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
             ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context),
                     precisionStep, boost, fieldType, nullValue, ignoreMalformed(context),
-                    provider, similarity, fieldDataSettings);
+                    provider, similarity, normsLoading, fieldDataSettings);
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }
@@ -117,10 +117,10 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
     private String nullValueAsString;
 
     protected ByteFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
-                              Byte nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
+                              Byte nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings) {
         super(names, precisionStep, boost, fieldType,
                 ignoreMalformed, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
-                new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, fieldDataSettings);
+                new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity, normsLoading, fieldDataSettings);
         this.nullValue = nullValue;
         this.nullValueAsString = nullValue == null ? null : nullValue.toString();
     }
@@ -377,4 +377,4 @@ public String numericAsString() {
             return Byte.toString(number);
         }
     }
-}
+}

src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java

@@ -207,7 +207,7 @@ private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name)
 
     public CompletionFieldMapper(Names names, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, PostingsFormatProvider provider, SimilarityProvider similarity, boolean payloads,
                                  boolean preserveSeparators, boolean preservePositionIncrements, int maxInputLength) {
-        super(names, 1.0f, Defaults.FIELD_TYPE, indexAnalyzer, searchAnalyzer, provider, similarity, null);
+        super(names, 1.0f, Defaults.FIELD_TYPE, indexAnalyzer, searchAnalyzer, provider, similarity, null, null);
         analyzingSuggestLookupProvider = new AnalyzingCompletionLookupProvider(preserveSeparators, false, preservePositionIncrements, payloads);
         this.completionPostingsFormatProvider = new CompletionPostingsFormatProvider("completion", provider, analyzingSuggestLookupProvider);
         this.preserveSeparators = preserveSeparators;

src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java

@@ -124,7 +124,7 @@ public DateFieldMapper build(BuilderContext context) {
             if (!locale.equals(dateTimeFormatter.locale())) {
                 dateTimeFormatter = new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale);
             }
-            DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter, precisionStep, boost, fieldType, nullValue, timeUnit, roundCeil, ignoreMalformed(context), provider, similarity, fieldDataSettings);
+            DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter, precisionStep, boost, fieldType, nullValue, timeUnit, roundCeil, ignoreMalformed(context), provider, similarity, normsLoading, fieldDataSettings);
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }
@@ -197,12 +197,12 @@ public static Locale parseLocale(String locale) {
 
     protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, float boost, FieldType fieldType,
                               String nullValue, TimeUnit timeUnit, boolean roundCeil, Explicit<Boolean> ignoreMalformed,
-                              PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
+                              PostingsFormatProvider provider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings) {
         super(names, precisionStep, boost, fieldType,
                 ignoreMalformed, new NamedAnalyzer("_date/" + precisionStep,
                 new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())),
                 new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())),
-                provider, similarity, fieldDataSettings);
+                provider, similarity, normsLoading, fieldDataSettings);
         this.dateTimeFormatter = dateTimeFormatter;
         this.nullValue = nullValue;
         this.timeUnit = timeUnit;
@@ -488,4 +488,4 @@ private long parseStringValue(String value) {
             }
         }
     }
-}
+}

src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java

@@ -89,7 +89,7 @@ public DoubleFieldMapper build(BuilderContext context) {
             fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
             DoubleFieldMapper fieldMapper = new DoubleFieldMapper(buildNames(context),
                     precisionStep, boost, fieldType, nullValue,
-                    ignoreMalformed(context), provider, similarity, fieldDataSettings);
+                    ignoreMalformed(context), provider, similarity, normsLoading, fieldDataSettings);
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }
@@ -118,10 +118,10 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
 
     protected DoubleFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
                                 Double nullValue, Explicit<Boolean> ignoreMalformed,
-                                PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
+                                PostingsFormatProvider provider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings) {
         super(names, precisionStep, boost, fieldType, ignoreMalformed,
                 NumericDoubleAnalyzer.buildNamedAnalyzer(precisionStep), NumericDoubleAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE),
-                provider, similarity, fieldDataSettings);
+                provider, similarity, normsLoading, fieldDataSettings);
         this.nullValue = nullValue;
         this.nullValueAsString = nullValue == null ? null : nullValue.toString();
     }

src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java

@@ -90,7 +90,7 @@ public FloatFieldMapper build(BuilderContext context) {
             fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
             FloatFieldMapper fieldMapper = new FloatFieldMapper(buildNames(context),
                     precisionStep, boost, fieldType, nullValue,
-                    ignoreMalformed(context), provider, similarity, fieldDataSettings);
+                    ignoreMalformed(context), provider, similarity, normsLoading, fieldDataSettings);
             fieldMapper.includeInAll(includeInAll);
             return fieldMapper;
         }
@@ -117,10 +117,10 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
     private String nullValueAsString;
 
     protected FloatFieldMapper(Names names, int precisionStep, float boost, FieldType fieldType,
-                               Float nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, @Nullable Settings fieldDataSettings) {
+                               Float nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity, Loading normsLoading, @Nullable Settings fieldDataSettings) {
         super(names, precisionStep, boost, fieldType, ignoreMalformed,
                 NumericFloatAnalyzer.buildNamedAnalyzer(precisionStep), NumericFloatAnalyzer.buildNamedAnalyzer(Integer.MAX_VALUE),
-                provider, similarity, fieldDataSettings);
+                provider, similarity, normsLoading, fieldDataSettings);
         this.nullValue = nullValue;
         this.nullValueAsString = nullValue == null ? null : nullValue.toString();
     }