Skip to content

Commit

Permalink
SOLR-13539:Fix multi value update of UUID, Enums, Bool and Binary fields
Browse files Browse the repository at this point in the history
  • Loading branch information
Thomas Wöckinger committed Aug 31, 2019
1 parent 66d7dff commit 973bcbb
Show file tree
Hide file tree
Showing 10 changed files with 1,240 additions and 54 deletions.
88 changes: 66 additions & 22 deletions solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java
View file
Expand Up @@ -16,6 +16,8 @@
*/
package org.apache.solr.client.solrj.embedded;

import static org.apache.solr.common.params.CommonParams.PATH;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
Expand All @@ -24,8 +26,8 @@
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.function.Supplier;

import com.google.common.base.Strings;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
Expand Down Expand Up @@ -57,7 +59,7 @@
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.servlet.SolrRequestParsers;

import static org.apache.solr.common.params.CommonParams.PATH;
import com.google.common.base.Strings;

/**
* SolrClient that connects directly to a CoreContainer.
Expand All @@ -69,6 +71,21 @@ public class EmbeddedSolrServer extends SolrClient {
protected final CoreContainer coreContainer;
protected final String coreName;
private final SolrRequestParsers _parser;
private final RequestWriterSupplier supplier;

public enum RequestWriterSupplier {
JavaBin(() -> new BinaryRequestWriter()), XML(() -> new RequestWriter());

private Supplier<RequestWriter> supplier;

private RequestWriterSupplier(final Supplier<RequestWriter> supplier) {
this.supplier = supplier;
}

public RequestWriter newRequestWriter() {
return supplier.get();
}
}

/**
* Create an EmbeddedSolrServer using a given solr home directory
Expand Down Expand Up @@ -112,6 +129,23 @@ public EmbeddedSolrServer(SolrCore core) {
* @param coreName the core to route requests to by default
*/
public EmbeddedSolrServer(CoreContainer coreContainer, String coreName) {
this(coreContainer, coreName, RequestWriterSupplier.JavaBin);
}

/**
* Create an EmbeddedSolrServer wrapping a CoreContainer.
* <p>
* Note that EmbeddedSolrServer will shutdown the wrapped CoreContainer when {@link #close()} is called.
*
* @param coreContainer
* the core container
* @param coreName
* the core to route requests to by default
* @param supplier
* the supplier used to create a {@link RequestWriter}
*/
public EmbeddedSolrServer(final CoreContainer coreContainer, final String coreName,
final RequestWriterSupplier supplier) {
if (coreContainer == null) {
throw new NullPointerException("CoreContainer instance required");
}
Expand All @@ -120,6 +154,7 @@ public EmbeddedSolrServer(CoreContainer coreContainer, String coreName) {
this.coreContainer = coreContainer;
this.coreName = coreName;
_parser = new SolrRequestParsers(null);
this.supplier = supplier;
}

// TODO-- this implementation sends the response to XML and then parses it.
Expand Down Expand Up @@ -237,35 +272,44 @@ public void writeResults(ResultContext ctx, JavaBinCodec codec) throws IOExcepti
}
}

private Set<ContentStream> getContentStreams(SolrRequest request) throws IOException {
private Set<ContentStream> getContentStreams(final SolrRequest<?> request) throws IOException {
if (request.getMethod() == SolrRequest.METHOD.GET) return null;
if (request instanceof ContentStreamUpdateRequest) {
ContentStreamUpdateRequest csur = (ContentStreamUpdateRequest) request;
Collection<ContentStream> cs = csur.getContentStreams();
final ContentStreamUpdateRequest csur = (ContentStreamUpdateRequest) request;
final Collection<ContentStream> cs = csur.getContentStreams();
if (cs != null) return new HashSet<>(cs);
}
RequestWriter.ContentWriter contentWriter = request.getContentWriter(CommonParams.JAVABIN_MIME);
final String cType = contentWriter == null ? CommonParams.JAVABIN_MIME : contentWriter.getContentType();

return Collections.singleton(new ContentStreamBase() {
final RequestWriter.ContentWriter contentWriter = request.getContentWriter(null);

String cType;
final BAOS baos = new BAOS();
if (contentWriter != null) {
contentWriter.write(baos);
cType = contentWriter.getContentType();
} else {
final RequestWriter rw = supplier.newRequestWriter();
cType = rw.getUpdateContentType();
rw.write(request, baos);
}

@Override
public InputStream getStream() throws IOException {
BAOS baos = new BAOS();
if (contentWriter != null) {
contentWriter.write(baos);
} else {
new BinaryRequestWriter().write(request, baos);
final byte[] buf = baos.toByteArray();
if (buf.length > 0) {
return Collections.singleton(new ContentStreamBase() {

@Override
public InputStream getStream() throws IOException {
return new ByteArrayInputStream(buf);
}
return new ByteArrayInputStream(baos.toByteArray());
}

@Override
public String getContentType() {
return cType;
@Override
public String getContentType() {
return cType;
}
});
}

}
});
return null;
}

private JavaBinCodec createJavaBinCodec(final StreamingResponseCallback callback, final BinaryResponseWriter.Resolver resolver) {
Expand Down
49 changes: 34 additions & 15 deletions solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
View file
Expand Up @@ -16,6 +16,20 @@
*/
package org.apache.solr.handler.loader;

import static org.apache.solr.common.params.CommonParams.ID;
import static org.apache.solr.common.params.CommonParams.NAME;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;

import javax.xml.parsers.SAXParserFactory;
import javax.xml.stream.FactoryConfigurationError;
import javax.xml.stream.XMLInputFactory;
Expand All @@ -27,19 +41,9 @@
import javax.xml.transform.dom.DOMResult;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.sax.SAXSource;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;

import com.google.common.collect.Lists;
import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.util.Constants;
import org.apache.solr.common.EmptyEntityResolver;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
Expand All @@ -49,6 +53,7 @@
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.StrUtils;
Expand All @@ -69,11 +74,10 @@
import org.xml.sax.InputSource;
import org.xml.sax.XMLReader;

import static org.apache.solr.common.params.CommonParams.ID;
import static org.apache.solr.common.params.CommonParams.NAME;

import com.google.common.collect.Lists;

public class XMLLoader extends ContentStreamLoader {

private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final AtomicBoolean WARNED_ABOUT_INDEX_TIME_BOOSTS = new AtomicBoolean();
static final XMLErrorLogger xmllog = new XMLErrorLogger(log);
Expand Down Expand Up @@ -406,6 +410,7 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti
boolean isNull = false;
boolean isLabeledChildDoc = false;
String update = null;
boolean binary = false;
Collection<SolrInputDocument> subDocs = null;
Map<String, Map<String, Object>> updateMap = null;
boolean complete = false;
Expand All @@ -429,7 +434,18 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti
break;
} else if ("field".equals(parser.getLocalName())) {
// should I warn in some text has been found too
Object v = isNull ? null : text.toString();
Object v;

if (isNull) {
v = null;
} else {
if (binary) {
v = Base64.base64ToByteArray(text.toString());
} else {
v = text.toString();
}
}

if (update != null) {
if (updateMap == null) updateMap = new HashMap<>();
Map<String, Object> extendedValues = updateMap.get(name);
Expand Down Expand Up @@ -492,6 +508,7 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti
}
update = null;
isNull = false;
binary = false;
String attrVal = "";
for (int i = 0; i < parser.getAttributeCount(); i++) {
attrName = parser.getAttributeLocalName(i);
Expand All @@ -509,6 +526,8 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti
isNull = StrUtils.parseBoolean(attrVal);
} else if ("update".equals(attrName)) {
update = attrVal;
} else if ("dt".equals(attrName)) {
binary = Constants.BINARY_BASE64.equals(attrVal);
} else {
log.warn("XML element <field> has invalid XML attr: " + attrName);
}
Expand Down
8 changes: 5 additions & 3 deletions solr/core/src/java/org/apache/solr/schema/BoolField.java
View file
Expand Up @@ -45,6 +45,7 @@
import org.apache.solr.search.QParser;
import org.apache.solr.search.function.OrdFieldSource;
import org.apache.solr.uninverting.UninvertingReader.Type;

/**
*
*/
Expand Down Expand Up @@ -208,7 +209,7 @@ public List<IndexableField> createFields(SchemaField field, Object value) {
}

@Override
public Object toNativeType(Object val) {
public Object toNativeType(final Object val) {
if (val instanceof CharSequence) {
return Boolean.valueOf(val.toString());
}
Expand Down Expand Up @@ -260,8 +261,8 @@ private int getOrdForDoc(int doc) throws IOException {
return -1;
}
}

@Override

public boolean boolVal(int doc) throws IOException {
return getOrdForDoc(doc) == trueOrd;
}
Expand Down Expand Up @@ -298,9 +299,10 @@ public boolean equals(Object o) {
}

private static final int hcode = OrdFieldSource.class.hashCode();

@Override
public int hashCode() {
return hcode + field.hashCode();
};
}

}
30 changes: 30 additions & 0 deletions solr/core/src/test-files/solr/collection1/conf/schema.xml
View file
Expand Up @@ -498,6 +498,14 @@
</analyzer>
</fieldType>
<fieldType name="severityType" class="${solr.tests.EnumFieldType}" enumsConfig="enumsConfig.xml" enumName="severity"/>

<fieldType name="binary" class="solr.BinaryField" />
<fieldType name="collation" class="solr.CollationField" language="en" />
<fieldType name="externalFile" class="solr.ExternalFileField" />
<fieldType name="icuCollation" class="solr.ICUCollationField" locale="en" />
<fieldType name="latLonPointSpatial" class="solr.LatLonPointSpatialField" />
<fieldType name="randomSort" class="solr.RandomSortField" />
<fieldType name="point" class="solr.PointType" subFieldSuffix="_coordinate" />

<fieldType name="sortable_text" class="solr.SortableTextField">
<analyzer>
Expand Down Expand Up @@ -632,6 +640,28 @@
<field name="dateRemove" type="date" indexed="true" stored="true" multiValued="true"/>
<field name="floatRemove" type="float" indexed="true" stored="true" multiValued="true"/>

<field name="binaryRemove" type="binary" indexed="true" stored="true" multiValued="true"/>
<field name="booleanRemove" type="boolean" indexed="true" stored="true" multiValued="true"/>
<field name="collationRemove" type="collation" indexed="true" stored="true" multiValued="true"/>
<field name="datePointRemove" type="pdate" indexed="true" stored="true" multiValued="true"/>
<field name="dateRangeRemove" type="dateRange" indexed="true" stored="true" multiValued="true"/>
<field name="doublePointRemove" type="pdouble" indexed="true" stored="true" multiValued="true"/>
<field name="externalFileRemove" type="externalFile" indexed="true" stored="true" multiValued="true"/>
<field name="floatPointRemove" type="pfloat" indexed="true" stored="true" multiValued="true"/>
<field name="icuCollationRemove" type="icuCollation" indexed="true" stored="true" multiValued="true"/>
<field name="intPointRemove" type="pint" indexed="true" stored="true" multiValued="true"/>
<field name="latLonPointSpatialRemove" type="latLonPointSpatial" indexed="true" stored="true" multiValued="true"/>
<field name="latLonRemove" type="location" indexed="true" stored="true" multiValued="true"/>
<field name="longPointRemove" type="plong" indexed="true" stored="true" multiValued="true"/>
<field name="point_0_coordinate" type="float" indexed="true" stored="true" multiValued="true"/>
<field name="point_1_coordinate" type="float" indexed="true" stored="true" multiValued="true"/>
<field name="pointRemove" type="point" indexed="true" stored="true" multiValued="true"/>
<field name="randomSortRemove" type="randomSort" indexed="true" stored="true" multiValued="true"/>
<field name="spatialRecursivePrefixTreeRemove" type="location_rpt" indexed="true" stored="true" multiValued="true"/>
<field name="stringRemove" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="textRemove" type="text" indexed="true" stored="true" multiValued="true"/>
<field name="uuidRemove" type="uuid" indexed="true" stored="true" multiValued="true"/>

<field name="nopositionstext" type="nopositions" indexed="true" stored="true"/>

<field name="tlong" type="tlong" indexed="true" stored="true"/>
Expand Down

0 comments on commit 973bcbb

Please sign in to comment.