Merge with master

vespa-engine · Jan 22, 2017 · 30cd111 · 30cd111
2 parents 76c5c6d + d7640ec
commit 30cd111
Show file tree

Hide file tree

Showing 100 changed files with 841 additions and 800 deletions.
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java
@@ -75,8 +75,7 @@ private void addField(String name, DataType type, SummaryTransform transform) {
             SummaryClassField sf = fields.get(name);
             if (!SummaryClassField.convertDataType(type, transform).equals(sf.getType())) {
                 deployLogger.log(Level.WARNING, "Conflicting definition of field " + name + ". " +
-                               "Declared as type " + sf.getType() + " and " +
-                               type);
+                               "Declared as type " + sf.getType() + " and " + type);
             }
         } else {
             fields.put(name, new SummaryClassField(name, type, transform));

diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java
@@ -33,11 +33,12 @@ public enum Type {
         LONGDATA("longdata"),
         XMLSTRING("xmlstring"),
         FEATUREDATA("featuredata"),
-        JSONSTRING("jsonstring");
+        JSONSTRING("jsonstring"),
+        TENSOR("tensor");
 
         private String name;
 
-        private Type(String name) {
+        Type(String name) {
             this.name = name;
         }
 
@@ -88,7 +89,7 @@ public static Type convertDataType(DataType fieldType, SummaryTransform transfor
         } else if (fval instanceof PredicateFieldValue) {
             return Type.STRING;
         } else if (fval instanceof TensorFieldValue) {
-            return Type.JSONSTRING;
+            return Type.TENSOR;
         } else if (fieldType instanceof CollectionDataType) {
             if (transform != null && transform.equals(SummaryTransform.POSITIONS)) {
                 return Type.XMLSTRING;

diff --git a/config-model/src/test/derived/tensor/summary.cfg b/config-model/src/test/derived/tensor/summary.cfg
@@ -1,27 +1,27 @@
-defaultsummaryid 1113059691
-classes[0].id 1113059691
-classes[0].name "default"
-classes[0].fields[0].name "f1"
-classes[0].fields[0].type "jsonstring"
-classes[0].fields[1].name "f3"
-classes[0].fields[1].type "jsonstring"
-classes[0].fields[2].name "f4"
-classes[0].fields[2].type "jsonstring"
-classes[0].fields[3].name "rankfeatures"
-classes[0].fields[3].type "featuredata"
-classes[0].fields[4].name "summaryfeatures"
-classes[0].fields[4].type "featuredata"
-classes[0].fields[5].name "documentid"
-classes[0].fields[5].type "longstring"
-classes[1].id 457955124
-classes[1].name "attributeprefetch"
-classes[1].fields[0].name "f2"
-classes[1].fields[0].type "jsonstring"
-classes[1].fields[1].name "f3"
-classes[1].fields[1].type "jsonstring"
-classes[1].fields[2].name "f4"
-classes[1].fields[2].type "jsonstring"
-classes[1].fields[3].name "rankfeatures"
-classes[1].fields[3].type "featuredata"
-classes[1].fields[4].name "summaryfeatures"
-classes[1].fields[4].type "featuredata"
+defaultsummaryid 289405525
+classes[].id 289405525
+classes[].name "default"
+classes[].fields[].name "f1"
+classes[].fields[].type "tensor"
+classes[].fields[].name "f3"
+classes[].fields[].type "tensor"
+classes[].fields[].name "f4"
+classes[].fields[].type "tensor"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "documentid"
+classes[].fields[].type "longstring"
+classes[].id 1860420340
+classes[].name "attributeprefetch"
+classes[].fields[].name "f2"
+classes[].fields[].type "tensor"
+classes[].fields[].name "f3"
+classes[].fields[].type "tensor"
+classes[].fields[].name "f4"
+classes[].fields[].type "tensor"
+classes[].fields[].name "rankfeatures"
+classes[].fields[].type "featuredata"
+classes[].fields[].name "summaryfeatures"
+classes[].fields[].type "featuredata"
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DataField.java
@@ -16,11 +16,11 @@
 import com.yahoo.data.access.simple.Value;
 import com.yahoo.data.access.Inspector;
 
-
 /**
- * @author  <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
 public class DataField extends DocsumField implements VariableLengthField {
+
     public DataField(String name) {
         super(name);
     }
@@ -67,4 +67,5 @@ public int sizeOfLength() {
     public Object convert(Inspector value) {
         return convert(value.asData(Value.empty().asData()));
     }
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumDefinitionSet.java
@@ -23,6 +23,7 @@
  * @author Bjørn Borud
  */
 public final class DocsumDefinitionSet {
+
     public static final int SLIME_MAGIC_ID = 0x55555555;
     private final static Logger log = Logger.getLogger(DocsumDefinitionSet.class.getName());
 
@@ -73,6 +74,7 @@ public final void lazyDecode(String summaryClass, byte[] data, FastHit hit) {
         buffer.order(ByteOrder.LITTLE_ENDIAN);
         long docsumClassId = buffer.getInt();
         if (docsumClassId != SLIME_MAGIC_ID) {
+            // TODO: Not used, remove   - bratseth 2017-01-016
             DocsumDefinition docsumDefinition = lookupDocsum(docsumClassId);
             Docsum docsum = new Docsum(docsumDefinition, data);
             hit.addSummary(docsum);

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java
@@ -13,35 +13,35 @@
 import com.yahoo.log.LogLevel;
 
 /**
- * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Bjørn Borud
+ * @author Steinar Knutsen
  */
 public abstract class DocsumField {
 
     private static final Logger log = Logger.getLogger(DocsumField.class.getName());
     private static FieldFactory fieldFactory;
 
     private static class FieldFactory {
+
         Map<String, Constructor<? extends DocsumField>> constructors = new HashMap<>();
 
-        void put(final String typename,
-                final Class<? extends DocsumField> fieldClass)
-                throws NoSuchMethodException, SecurityException
-        {
-            final Constructor<? extends DocsumField> constructor = fieldClass.getConstructor(String.class);
+        void put(String typename, Class<? extends DocsumField> fieldClass) 
+                throws NoSuchMethodException, SecurityException {
+            Constructor<? extends DocsumField> constructor = fieldClass.getConstructor(String.class);
             constructors.put(typename, constructor);
         }
 
-        DocsumField create(final String typename, final String name, final LegacyEmulationConfig emulConfig)
+        DocsumField create(String typename, String name, LegacyEmulationConfig emulConfig)
                 throws InstantiationException, IllegalAccessException,
-                IllegalArgumentException, InvocationTargetException
-        {
+                       IllegalArgumentException, InvocationTargetException {
             DocsumField f = constructors.get(typename).newInstance(name);
             f.emulConfig = emulConfig;
             return f;
         }
     }
+
     private LegacyEmulationConfig emulConfig;
+
     final LegacyEmulationConfig getEmulConfig() { return emulConfig; }
 
     static {
@@ -61,27 +61,27 @@ DocsumField create(final String typename, final String name, final LegacyEmulati
             fieldFactory.put("jsonstring", StructDataField.class);
             fieldFactory.put("featuredata", FeatureDataField.class);
             fieldFactory.put("xmlstring", XMLField.class);
-        } catch (final Exception e) {
-            log.log(LogLevel.ERROR,
-                    "Could not initialize docsum decoding properly.", e);
+            fieldFactory.put("tensor", TensorField.class);
+        } catch (Exception e) {
+            log.log(LogLevel.ERROR, "Could not initialize docsum decoding properly.", e);
         }
     }
 
     protected String name;
 
-    protected DocsumField(final String name) {
+    protected DocsumField(String name) {
         this.name = name;
     }
 
     /* for unit test only */
-    static DocsumField create(final String name, final String typename) {
+    static DocsumField create(String name, String typename) {
         return create(name, typename, new LegacyEmulationConfig());
     }
 
-    public static DocsumField create(final String name, final String typename, LegacyEmulationConfig emulConfig) {
+    public static DocsumField create(String name, String typename, LegacyEmulationConfig emulConfig) {
         try {
             return fieldFactory.create(typename, name, emulConfig);
-        } catch (final Exception e) {
+        } catch (Exception e) {
             throw new RuntimeException("Unknown field type '" + typename + "'", e);
         }
     }
@@ -90,7 +90,7 @@ public String getName() {
         return name;
     }
 
-    public boolean isCompressed(final ByteBuffer b) {
+    public boolean isCompressed(ByteBuffer b) {
         return false;
     }
 
@@ -106,8 +106,8 @@ public boolean isCompressed(final ByteBuffer b) {
     public abstract Object decode(ByteBuffer b);
 
     /**
-     * Get the number of bytes this field occupies in the given buffer and set
-     * the position of the first byte after this field.
+     * Get the number of bytes this field occupies in the given buffer 
+     * AND SET(!) the position to the first byte after this field.
      */
     public abstract int getLength(ByteBuffer b);
 
@@ -116,4 +116,5 @@ public boolean isCompressed(final ByteBuffer b) {
      * for this field.
      **/
     public abstract Object convert(Inspector value);
+
 }
diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java
@@ -174,8 +174,7 @@ public void setPartId(int partId, int rowBits) {
     }
 
     /**
-     *
-     * @param useRowInIndexUri Sets whether to use the row in the index uri. See FastSearcher for details.
+     * Sets whether to use the row in the index uri. See FastSearcher for details.
      */
     public void setUseRowInIndexUri(boolean useRowInIndexUri) {
         this.useRowInIndexUri = useRowInIndexUri;
@@ -189,7 +188,7 @@ public int getColumn() {
     }
 
     /**
-     *  @return the row number where this hit originated, or 0 if not known
+     *  Returns the row number where this hit originated, or 0 if not known
      * */
     public int getRow() {
         if (rowBits == 0) {
@@ -214,17 +213,20 @@ public int getRow() {
      * filled returns the following types, even when the field has no actual value:</p>
      *
      * <ul>
-     *     <li><b>Dynamic summary string fields</b>: A Java String before JuniperSearcher and a HitField after.</li>
-     *     <li><b>string/uri/content</b>: A Java String.<br>
+     *     <li><b>string and uri fields</b>: A Java String.<br>
      *     The empty string ("") if no value is assigned in the document.
      *
+     *     <li><b>Dynamic summary string fields</b>: A Java String before JuniperSearcher and a HitField after.</li>
+     *     
      *     <li><b>Numerics</b>: The corresponding numeric Java type.<br>
      *     If the field has <i>no value</i> assigned in the document,
      *     the special numeric {@link com.yahoo.search.result.NanNumber#NaN} is returned.
      *
-     *     <li><b>raw</b>: A {@link com.yahoo.prelude.hitfield.RawData} instance
+     *     <li><b>raw fields</b>: A {@link com.yahoo.prelude.hitfield.RawData} instance
+     *
+     *     <li><b>tensor fields</b>: A {@link com.yahoo.tensor.Tensor} instance
      *
-     *     <li><b>multivalue fields</b>: A {@link com.yahoo.prelude.hitfield.JSONString} instance
+     *     <li><b>multivalue fields</b>: A {@link com.yahoo.data.access.Inspector} instance
      * </ul>
      */
     @Override
@@ -277,10 +279,7 @@ void addSummary(DocsumDefinition docsumDef, Inspector value) {
         for (DocsumField field : docsumDef.getFields()) {
             String fieldName = field.getName();
             if (value.type() == Type.STRING &&
-                (field instanceof LongstringField ||
-                 field instanceof StringField ||
-                 field instanceof XMLField))
-            {
+                (field instanceof LongstringField || field instanceof StringField || field instanceof XMLField)) {
                 setDocsumFieldIfNotPresent(fieldName, new LazyString(field, value));
             } else {
                 Inspector f = value.field(fieldName);

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/JSONField.java
@@ -1,7 +1,6 @@
 // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.fastsearch;
 
-
 import java.nio.ByteBuffer;
 
 import com.yahoo.io.SlowInflate;
@@ -10,13 +9,13 @@
 import com.yahoo.data.access.*;
 import com.yahoo.data.access.simple.Value;
 
-
 /**
- * Class representing a JSON string field in the result set
+ * A hit field containing JSON structured data
  *
- * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a>
+ * @author Steinar Knutsen
  */
 public class JSONField extends DocsumField implements VariableLengthField {
+
     public JSONField(String name) {
         super(name);
     }
@@ -104,9 +103,7 @@ Inspector stringify(Inspector value) {
         }
     }
 
-    private static class ArrConv extends CompatibilityConverter
-        implements ArrayTraverser
-    {
+    private static class ArrConv extends CompatibilityConverter implements ArrayTraverser {
         @Override
         public void entry(int idx, Inspector value) {
             target.add(stringify(value));

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongdataField.java
@@ -5,19 +5,18 @@
  */
 package com.yahoo.prelude.fastsearch;
 
-
 import java.nio.ByteBuffer;
 
 import com.yahoo.io.SlowInflate;
 import com.yahoo.prelude.hitfield.RawData;
 import com.yahoo.data.access.simple.Value;
 import com.yahoo.data.access.Inspector;
 
-
 /**
- * @author  <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
 public class LongdataField extends DocsumField implements VariableLengthField {
+
     public LongdataField(String name) {
         super(name);
     }

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/LongstringField.java
@@ -5,18 +5,17 @@
  */
 package com.yahoo.prelude.fastsearch;
 
-
 import java.nio.ByteBuffer;
 
 import com.yahoo.io.SlowInflate;
 import com.yahoo.text.Utf8;
 import com.yahoo.data.access.Inspector;
 
-
 /**
- * @author  <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
 public class LongstringField extends DocsumField implements VariableLengthField {
+
     public LongstringField(String name) {
         super(name);
     }

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/ShortField.java
@@ -12,9 +12,8 @@
 import com.yahoo.data.access.Inspector;
 
 /**
- * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
-
 public class ShortField extends DocsumField {
     static final short EMPTY_VALUE = Short.MIN_VALUE;
 

diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/StringField.java
@@ -13,9 +13,10 @@
 
 
 /**
- * @author <a href="mailto:borud@yahoo-inc.com">Bj\u00f8rn Borud</a>
+ * @author Bjørn Borud
  */
 public class StringField extends DocsumField implements VariableLengthField {
+
     public StringField(String name) {
         super(name);
     }
@@ -59,4 +60,5 @@ public int sizeOfLength() {
     public Object convert(Inspector value) {
         return value.asString("");
     }
+
 }