SOLR-14454: support for UTF-8 (string) types with DocValuesType.BINARY

solr/core/src/java/org/apache/solr/handler/export/BinaryStringFieldWriter.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRefBuilder;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.ByteArrayUtf8CharSequence;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+
+class BinaryStringFieldWriter extends FieldWriter {
+  private final String field;
+  private final FieldType fieldType;
+  private final CharsRefBuilder cref = new CharsRefBuilder();
+  private final ByteArrayUtf8CharSequence utf8 = new ByteArrayUtf8CharSequence(new byte[0], 0, 0) {
+    @Override
+    public String toString() {
+      String str = super.utf16;
+      if (str != null) return str;
+      fieldType.indexedToReadable(new BytesRef(super.buf, super.offset, super.length), cref);
+      str = cref.toString();
+      super.utf16 = str;
+      return str;
+    }
+  };
+
+  // for reusing BinaryDocValues across invocations, where possible
+  private LeafReader lastReader;
+  private int lastDocId = -1;
+  private BinaryDocValues reuseDVs;
+
+  public BinaryStringFieldWriter(String field, SchemaField schemaField, FieldType fieldType) {
+    this.field = field;
+    this.fieldType = fieldType;
+  }
+
+  private BinaryDocValues getBinaryDocValues(int docId, LeafReader reader) throws IOException {
+    if (lastReader != reader || docId < lastDocId) {
+      lastReader = reader;
+      reuseDVs = DocValues.getBinary(reader, field);
+    }
+    lastDocId = docId;
+    return reuseDVs;
+  }
+
+  public boolean write(SortDoc sortDoc, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+    BytesRef ref;
+    SortValue sortValue = sortDoc.getSortValue(this.field);
+    if (sortValue != null) {
+      if (sortValue.isPresent()) {
+        ref = (BytesRef) sortValue.getCurrentValue();
+      } else { //empty-value
+        return false;
+      }
+    } else {
+      // field is not part of 'sort' param, but part of 'fl' param
+      final BinaryDocValues bdv = getBinaryDocValues(sortDoc.docId, reader);
+      if (!bdv.advanceExact(sortDoc.docId)) {
+        return false;
+      }
+      ref = bdv.binaryValue();
+    }
+
+    if (ew instanceof JavaBinCodec.BinEntryWriter) {
+      ew.put(this.field, utf8.reset(ref.bytes, ref.offset, ref.length, null));
+    } else {
+      fieldType.indexedToReadable(ref, cref);
+      String v = cref.toString();
+      ew.put(this.field, v);
+    }
+    return true;
+  }
+}

solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java

@@ -24,8 +24,12 @@
 import java.io.PrintWriter;
 import java.lang.invoke.MethodHandles;
 import java.nio.charset.StandardCharsets;
+import java.util.EnumSet;
 import java.util.List;
 
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedDocValues;
@@ -318,6 +322,7 @@ protected void writeDoc(SortDoc sortDoc,
 
   protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searcher) throws IOException {
     IndexSchema schema = searcher.getSchema();
+    final FieldInfos fieldInfos = searcher.getFieldInfos();
     FieldWriter[] writers = new FieldWriter[fields.length];
     for (int i = 0; i < fields.length; i++) {
       String field = fields[i];
@@ -332,6 +337,11 @@ protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searc
       if (!schemaField.hasDocValues()) {
         throw new IOException(schemaField + " must have DocValues to use this feature.");
       }
+      final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+      if (fieldInfo == null) {
+        writers[i] = EMPTY_FIELD_WRITER;
+        continue;
+      }
       boolean multiValued = schemaField.multiValued();
       FieldType fieldType = schemaField.getType();
 
@@ -363,11 +373,19 @@ protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searc
         } else {
           writers[i] = new DoubleFieldWriter(field);
         }
-      } else if (fieldType instanceof StrField || fieldType instanceof SortableTextField) {
-        if (multiValued) {
-          writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
-        } else {
-          writers[i] = new StringFieldWriter(field, fieldType);
+      } else if (fieldType.isUtf8Field()) {
+        switch (fieldInfo.getDocValuesType()) {
+          case SORTED_SET:
+            writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
+            break;
+          case SORTED:
+            writers[i] = new StringFieldWriter(field, fieldType);
+            break;
+          case BINARY:
+            writers[i] = new BinaryStringFieldWriter(field, schemaField, fieldType);
+            break;
+          default:
+            throw new IOException("Utf8 export fields support DocValuesTypes "+EnumSet.of(DocValuesType.SORTED,DocValuesType.SORTED_SET,DocValuesType.BINARY)+"; found: "+fieldInfo.getDocValuesType());
         }
       } else if (fieldType instanceof DateValueFieldType) {
         if (multiValued) {
@@ -388,6 +406,13 @@ protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searc
     return writers;
   }
 
+  private static final FieldWriter EMPTY_FIELD_WRITER = new FieldWriter() {
+    @Override
+    public boolean write(SortDoc sortDoc, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException {
+      return false;
+    }
+  };
+
   private SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IOException {
     SortValue[] sortValues = new SortValue[sortFields.length];
     IndexSchema schema = searcher.getSchema();