From 494bc0c1ec9ee6a588afaf16dbdc80c89c62355a Mon Sep 17 00:00:00 2001 From: Moshe Date: Thu, 2 Aug 2018 19:03:13 +0300 Subject: [PATCH 01/13] SOLR-12485: support labelled children in xml documents --- .../apache/solr/handler/loader/XMLLoader.java | 14 ++++ .../solr/update/AddBlockUpdateTest.java | 67 +++++++++++++++++++ .../solr/client/solrj/util/ClientUtils.java | 4 +- 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java index a07aff24a200..e9f5db845169 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java @@ -463,6 +463,20 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti text.setLength(0); String localName = parser.getLocalName(); if ("doc".equals(localName)) { + String docKey = null; + for(int i = 0; i < parser.getAttributeCount(); ++i) { + attrName = parser.getAttributeLocalName(i); + if("name".equals(attrName)) { + docKey = parser.getAttributeValue(i); + } + } + if(docKey != null) { + if(!doc.containsKey(docKey)) { + doc.setField(docKey, Lists.newArrayList()); + } + doc.addField(docKey, readDoc(parser)); + break; + } if (subDocs == null) subDocs = Lists.newArrayList(); subDocs.add(readDoc(parser)); diff --git a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java index 191316639502..09749fe6d12d 100644 --- a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java +++ b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java @@ -501,6 +501,73 @@ public void testXML() throws IOException, XMLStreamException { } + @Test + public void testXMLLabeledChildren() throws IOException, XMLStreamException { + UpdateRequest req = new UpdateRequest(); + + List docs = new ArrayList<>(); + + String xml_doc1 = + "" + + " 1" + + " X" + + " " + + " 2" + + " y" + + ""+ + " " + + " 3" + + " z" + + ""+ + ""; + + String xml_doc2 = + "" + + " 4" + + " A" + + " " + + " 5" + + " b" + + ""+ + " " + + " 6" + + " c" + + ""+ + ""; + + XMLStreamReader parser = + inputFactory.createXMLStreamReader( new StringReader( xml_doc1 ) ); + parser.next(); // read the START document... + //null for the processor is all right here + XMLLoader loader = new XMLLoader(); + SolrInputDocument document1 = loader.readDoc( parser ); + + XMLStreamReader parser2 = + inputFactory.createXMLStreamReader( new StringReader( xml_doc2 ) ); + parser2.next(); // read the START document... + //null for the processor is all right here + //XMLLoader loader = new XMLLoader(); + SolrInputDocument document2 = loader.readDoc( parser2 ); + + + docs.add(document1); + docs.add(document2); + + Collections.shuffle(docs, random()); + req.add(docs); + + RequestWriter requestWriter = new RequestWriter(); + OutputStream os = new ByteArrayOutputStream(); + requestWriter.write(req, os); + assertBlockU(os.toString()); + assertU(commit()); + + final SolrIndexSearcher searcher = getSearcher(); + assertSingleParentOf(searcher, one("yz"), "X"); + assertSingleParentOf(searcher, one("bc"), "A"); + + } + @Test public void testJavaBinCodecNestedRelation() throws IOException { SolrInputDocument topDocument = new SolrInputDocument(); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index 54986db5b0cb..a3ceca1b0152 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -72,7 +72,9 @@ public static void writeXML( SolrInputDocument doc, Writer writer ) throws IOExc for( Object v : field ) { String update = null; - if (v instanceof Map) { + if(v instanceof SolrInputDocument) { + writeXML((SolrInputDocument) v ,writer); + } else if (v instanceof Map) { // currently only supports a single value for (Entry entry : ((Map)v).entrySet()) { update = entry.getKey().toString(); From f6dd5f98c43beacae7dc79a533038aaa0f0b99e5 Mon Sep 17 00:00:00 2001 From: Moshe Date: Sun, 5 Aug 2018 11:11:56 +0300 Subject: [PATCH 02/13] SOLR-12485: support labelled children as field values in xml documents --- .../apache/solr/handler/loader/XMLLoader.java | 21 +++++----- .../solr/update/AddBlockUpdateTest.java | 38 +++++++++++-------- .../solr/client/solrj/util/ClientUtils.java | 26 ++++++++++++- .../java/org/apache/solr/common/util/XML.java | 14 +++++-- 4 files changed, 67 insertions(+), 32 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java index e9f5db845169..3c40465f87dc 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java @@ -453,6 +453,10 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti } break; } + if(name == null) { + // do not add empty data if a labeled child document was added + break; + } doc.addField(name, v); // field is over name = null; @@ -463,18 +467,13 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti text.setLength(0); String localName = parser.getLocalName(); if ("doc".equals(localName)) { - String docKey = null; - for(int i = 0; i < parser.getAttributeCount(); ++i) { - attrName = parser.getAttributeLocalName(i); - if("name".equals(attrName)) { - docKey = parser.getAttributeValue(i); - } - } - if(docKey != null) { - if(!doc.containsKey(docKey)) { - doc.setField(docKey, Lists.newArrayList()); + if(name != null) { + if(!doc.containsKey(name)) { + doc.setField(name, Lists.newArrayList()); } - doc.addField(docKey, readDoc(parser)); + doc.addField(name, readDoc(parser)); + // signal to prevent spaces after doc from being added + name = null; break; } if (subDocs == null) diff --git a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java index 09749fe6d12d..e20a0e7192bf 100644 --- a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java +++ b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java @@ -511,28 +511,34 @@ public void testXMLLabeledChildren() throws IOException, XMLStreamException { "" + " 1" + " X" + - " " + - " 2" + - " y" + - ""+ - " " + - " 3" + - " z" + - ""+ + " " + + " " + + " 2" + + " y" + + " "+ + " " + + " 3" + + " z" + + " " + + " " + ""; String xml_doc2 = "" + " 4" + " A" + - " " + - " 5" + - " b" + - ""+ - " " + - " 6" + - " c" + - ""+ + " " + + " " + + " 5" + + " b" + + " "+ + " " + + " " + + " " + + " 6" + + " c" + + " " + + " " + ""; XMLStreamReader parser = diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index a3ceca1b0152..cc056aabd01f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -16,16 +16,21 @@ */ package org.apache.solr.client.solrj.util; +import java.io.BufferedWriter; import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; import java.io.StringWriter; import java.io.Writer; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.Map; import java.util.Map.Entry; +import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.apache.solr.common.cloud.Slice; @@ -73,7 +78,7 @@ public static void writeXML( SolrInputDocument doc, Writer writer ) throws IOExc String update = null; if(v instanceof SolrInputDocument) { - writeXML((SolrInputDocument) v ,writer); + writeVal(writer, name, v , null); } else if (v instanceof Map) { // currently only supports a single value for (Entry entry : ((Map)v).entrySet()) { @@ -116,17 +121,34 @@ private static void writeVal(Writer writer, String name, Object v, String update if (update == null) { if (v != null) { - XML.writeXML(writer, "field", v.toString(), "name", name ); + if(v instanceof SolrInputDocument) { + OutputStream os = SolrInputDocumentXmlOutput((SolrInputDocument) v); + XML.writeXML(writer, "field", false, os.toString(), "name", name); + } else { + XML.writeXML(writer, "field", v.toString(), "name", name ); + } } } else { if (v == null) { XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true); } else { + if(v instanceof SolrInputDocument) { + OutputStream os = SolrInputDocumentXmlOutput((SolrInputDocument) v); + XML.writeXML(writer, "field", false, os.toString(), "name", name, "update", update); + } XML.writeXML(writer, "field", v.toString(), "name", name, "update", update); } } } + private static OutputStream SolrInputDocumentXmlOutput(SolrInputDocument v) throws IOException { + OutputStream os = new ByteArrayOutputStream(); + BufferedWriter childDocWriter = new BufferedWriter(new OutputStreamWriter(os, StandardCharsets.UTF_8)); + writeXML(v, childDocWriter); + childDocWriter.flush(); + return os; + } + public static String toXML( SolrInputDocument doc ) { diff --git a/solr/solrj/src/java/org/apache/solr/common/util/XML.java b/solr/solrj/src/java/org/apache/solr/common/util/XML.java index c6e520523e8d..5d94b726497f 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/XML.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/XML.java @@ -126,8 +126,8 @@ public final static void writeUnescapedXML(Writer out, String tag, String val, O } } - /** escapes character data in val */ - public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException { + /** escapes character data in val if shouldEscape is true*/ + public final static void writeXML(Writer out, String tag, boolean shouldEscape, String val, Object... attrs) throws IOException { out.write('<'); out.write(tag); for (int i=0; i'); } else { out.write('>'); - escapeCharData(val,out); + if(shouldEscape) { + escapeCharData(val,out); + } else { + out.write(val); + } out.write('<'); out.write('/'); out.write(tag); @@ -151,6 +155,10 @@ public final static void writeXML(Writer out, String tag, String val, Object... } } + public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException { + writeXML(out, tag, true, val, attrs); + } + /** escapes character data in val */ public static void writeXML(Writer out, String tag, String val, Map attrs) throws IOException { out.write('<'); From c1e0943faa19ffd7626c8f2ead7c2c273997b54d Mon Sep 17 00:00:00 2001 From: Moshe Date: Sun, 5 Aug 2018 15:53:02 +0300 Subject: [PATCH 03/13] SOLR-12485: fix labelled children as field values in xml documents (no whitespace added) --- .../org/apache/solr/handler/loader/XMLLoader.java | 12 +++++++----- .../org/apache/solr/update/AddBlockUpdateTest.java | 10 +++++++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java index 3c40465f87dc..825fc73922cd 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java @@ -404,6 +404,7 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti StringBuilder text = new StringBuilder(); String name = null; boolean isNull = false; + boolean isLabeledChildDoc = false; String update = null; Collection subDocs = null; Map> updateMap = null; @@ -453,11 +454,12 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti } break; } - if(name == null) { + if(!isLabeledChildDoc){ // do not add empty data if a labeled child document was added - break; + doc.addField(name, v); + } else { + isLabeledChildDoc = false; } - doc.addField(name, v); // field is over name = null; } @@ -468,12 +470,12 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti String localName = parser.getLocalName(); if ("doc".equals(localName)) { if(name != null) { + // flag to prevent spaces after doc from being added + isLabeledChildDoc = true; if(!doc.containsKey(name)) { doc.setField(name, Lists.newArrayList()); } doc.addField(name, readDoc(parser)); - // signal to prevent spaces after doc from being added - name = null; break; } if (subDocs == null) diff --git a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java index e20a0e7192bf..272708b81027 100644 --- a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java +++ b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java @@ -510,6 +510,7 @@ public void testXMLLabeledChildren() throws IOException, XMLStreamException { String xml_doc1 = "" + " 1" + + " " + " X" + " " + " " + @@ -520,7 +521,7 @@ public void testXMLLabeledChildren() throws IOException, XMLStreamException { " 3" + " z" + " " + - " " + + " " + ""; String xml_doc2 = @@ -555,6 +556,13 @@ public void testXMLLabeledChildren() throws IOException, XMLStreamException { //XMLLoader loader = new XMLLoader(); SolrInputDocument document2 = loader.readDoc( parser2 ); + assertFalse(document1.hasChildDocuments()); + assertEquals(document1.toString(), sdoc("id", "1", "empty_s", "", "parent_s", "X", "test", + sdocs(sdoc("id", "2", "child_s", "y"), sdoc("id", "3", "child_s", "z"))).toString()); + + assertFalse(document2.hasChildDocuments()); + assertEquals(document2.toString(), sdoc("id", "4", "parent_s", "A", "test", + sdocs(sdoc("id", "5", "child_s", "b"), sdoc("id", "6", "child_s", "c"))).toString()); docs.add(document1); docs.add(document2); From 8d32afa1a587780228963dcc26014b02066bd1ac Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 6 Aug 2018 08:26:22 +0300 Subject: [PATCH 04/13] SOLR-12485: add multi level XML loader test --- .../solr/update/AddBlockUpdateTest.java | 67 +++++++++++++++++++ .../solr/client/solrj/util/ClientUtils.java | 13 ++-- .../java/org/apache/solr/common/util/XML.java | 14 +--- 3 files changed, 77 insertions(+), 17 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java index 272708b81027..97924ec17300 100644 --- a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java +++ b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java @@ -501,6 +501,73 @@ public void testXML() throws IOException, XMLStreamException { } + @Test + public void testXMLMultiLevelLabeledChildren() throws IOException, XMLStreamException { + String xml_doc1 = + "" + + " 1" + + " " + + " X" + + " " + + " " + + " 2" + + " y" + + " " + + " " + + " 3" + + " z" + + " " + + " " + + ""; + + String xml_doc2 = + "" + + " 4" + + " A" + + " " + + " " + + " 5" + + " b" + + " " + + " " + + " 7" + + " d" + + " " + + " " + + " " + + " " + + " " + + " " + + " 6" + + " c" + + " " + + " " + + ""; + + XMLStreamReader parser = + inputFactory.createXMLStreamReader(new StringReader(xml_doc1)); + parser.next(); // read the START document... + //null for the processor is all right here + XMLLoader loader = new XMLLoader(); + SolrInputDocument document1 = loader.readDoc(parser); + + XMLStreamReader parser2 = + inputFactory.createXMLStreamReader(new StringReader(xml_doc2)); + parser2.next(); // read the START document... + //null for the processor is all right here + //XMLLoader loader = new XMLLoader(); + SolrInputDocument document2 = loader.readDoc(parser2); + + assertFalse(document1.hasChildDocuments()); + assertEquals(document1.toString(), sdoc("id", "1", "empty_s", "", "parent_s", "X", "test", + sdocs(sdoc("id", "2", "child_s", "y"), sdoc("id", "3", "child_s", "z"))).toString()); + + assertFalse(document2.hasChildDocuments()); + assertEquals(document2.toString(), sdoc("id", "4", "parent_s", "A", "test", + sdocs(sdoc("id", "5", "child_s", "b", "grandChild", Collections.singleton(sdoc("id", "7", "child_s", "d"))), + sdoc("id", "6", "child_s", "c"))).toString()); + } + @Test public void testXMLLabeledChildren() throws IOException, XMLStreamException { UpdateRequest req = new UpdateRequest(); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index cc056aabd01f..0c6b7a7b87ce 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -122,8 +122,8 @@ private static void writeVal(Writer writer, String name, Object v, String update if (update == null) { if (v != null) { if(v instanceof SolrInputDocument) { - OutputStream os = SolrInputDocumentXmlOutput((SolrInputDocument) v); - XML.writeXML(writer, "field", false, os.toString(), "name", name); + OutputStream os = solrInputDocumentXmlOutput((SolrInputDocument) v); + XML.writeUnescapedXML(writer, "field", os.toString(), "name", name); } else { XML.writeXML(writer, "field", v.toString(), "name", name ); } @@ -133,15 +133,16 @@ private static void writeVal(Writer writer, String name, Object v, String update XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true); } else { if(v instanceof SolrInputDocument) { - OutputStream os = SolrInputDocumentXmlOutput((SolrInputDocument) v); - XML.writeXML(writer, "field", false, os.toString(), "name", name, "update", update); + OutputStream os = solrInputDocumentXmlOutput((SolrInputDocument) v); + XML.writeUnescapedXML(writer, "field", os.toString(), "name", name, "update", update); + } else { + XML.writeXML(writer, "field", v.toString(), "name", name, "update", update); } - XML.writeXML(writer, "field", v.toString(), "name", name, "update", update); } } } - private static OutputStream SolrInputDocumentXmlOutput(SolrInputDocument v) throws IOException { + private static OutputStream solrInputDocumentXmlOutput(SolrInputDocument v) throws IOException { OutputStream os = new ByteArrayOutputStream(); BufferedWriter childDocWriter = new BufferedWriter(new OutputStreamWriter(os, StandardCharsets.UTF_8)); writeXML(v, childDocWriter); diff --git a/solr/solrj/src/java/org/apache/solr/common/util/XML.java b/solr/solrj/src/java/org/apache/solr/common/util/XML.java index 5d94b726497f..c6e520523e8d 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/XML.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/XML.java @@ -126,8 +126,8 @@ public final static void writeUnescapedXML(Writer out, String tag, String val, O } } - /** escapes character data in val if shouldEscape is true*/ - public final static void writeXML(Writer out, String tag, boolean shouldEscape, String val, Object... attrs) throws IOException { + /** escapes character data in val */ + public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException { out.write('<'); out.write(tag); for (int i=0; i'); } else { out.write('>'); - if(shouldEscape) { - escapeCharData(val,out); - } else { - out.write(val); - } + escapeCharData(val,out); out.write('<'); out.write('/'); out.write(tag); @@ -155,10 +151,6 @@ public final static void writeXML(Writer out, String tag, boolean shouldEscape, } } - public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException { - writeXML(out, tag, true, val, attrs); - } - /** escapes character data in val */ public static void writeXML(Writer out, String tag, String val, Map attrs) throws IOException { out.write('<'); From 7602895e65ce28627a2fa373663f00344d193b97 Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 6 Aug 2018 11:33:59 +0300 Subject: [PATCH 05/13] SOLR-12485: writeUnescapedXML use lambda to write val --- .../solr/client/solrj/util/ClientUtils.java | 16 ++++++---------- .../java/org/apache/solr/common/util/XML.java | 12 ++++++++++-- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index 0c6b7a7b87ce..87bafe406526 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -122,8 +122,8 @@ private static void writeVal(Writer writer, String name, Object v, String update if (update == null) { if (v != null) { if(v instanceof SolrInputDocument) { - OutputStream os = solrInputDocumentXmlOutput((SolrInputDocument) v); - XML.writeUnescapedXML(writer, "field", os.toString(), "name", name); + final SolrInputDocument currChildDoc = (SolrInputDocument) v; + XML.writeUnescapedXML(writer, "field", (writer1) -> solrInputDocumentXmlOutput(currChildDoc, writer1), "name", name); } else { XML.writeXML(writer, "field", v.toString(), "name", name ); } @@ -133,8 +133,8 @@ private static void writeVal(Writer writer, String name, Object v, String update XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true); } else { if(v instanceof SolrInputDocument) { - OutputStream os = solrInputDocumentXmlOutput((SolrInputDocument) v); - XML.writeUnescapedXML(writer, "field", os.toString(), "name", name, "update", update); + final SolrInputDocument currChildDoc = (SolrInputDocument) v; + XML.writeUnescapedXML(writer, "field", (writer1) -> solrInputDocumentXmlOutput(currChildDoc, writer1), "name", name); } else { XML.writeXML(writer, "field", v.toString(), "name", name, "update", update); } @@ -142,12 +142,8 @@ private static void writeVal(Writer writer, String name, Object v, String update } } - private static OutputStream solrInputDocumentXmlOutput(SolrInputDocument v) throws IOException { - OutputStream os = new ByteArrayOutputStream(); - BufferedWriter childDocWriter = new BufferedWriter(new OutputStreamWriter(os, StandardCharsets.UTF_8)); - writeXML(v, childDocWriter); - childDocWriter.flush(); - return os; + private static void solrInputDocumentXmlOutput(SolrInputDocument v, Writer writer) throws IOException { + writeXML(v, writer); } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/XML.java b/solr/solrj/src/java/org/apache/solr/common/util/XML.java index c6e520523e8d..f7d6773ea1da 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/XML.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/XML.java @@ -101,8 +101,12 @@ public final static void writeXML(Writer out, String tag, String val) throws IOE } } - /** does NOT escape character data in val, must already be valid XML */ public final static void writeUnescapedXML(Writer out, String tag, String val, Object... attrs) throws IOException { + writeUnescapedXML(out, tag, (writer1) -> writer1.write(val), attrs); + } + + /** does NOT escape character data in val, must already be valid XML */ + public final static void writeUnescapedXML(Writer out, String tag, Writable val, Object... attrs) throws IOException { out.write('<'); out.write(tag); for (int i=0; i'); } else { out.write('>'); - out.write(val); + val.write(out); out.write('<'); out.write('/'); out.write(tag); @@ -203,4 +207,8 @@ private static void escape(String str, Writer out, String[] escapes) throws IOEx out.write(ch); } } + + public interface Writable { + void write(Writer w) throws IOException ; + } } From 79af6ca06d529982c84e8a6d771b67fa806fd6b4 Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 6 Aug 2018 13:27:00 +0300 Subject: [PATCH 06/13] SOLR-12485: writeUnescapedXML use lambda to write val even when escaped --- .../solr/client/solrj/util/ClientUtils.java | 28 ++++++++----------- .../java/org/apache/solr/common/util/XML.java | 4 +-- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index 87bafe406526..5dfaf8144a96 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -16,21 +16,16 @@ */ package org.apache.solr.client.solrj.util; -import java.io.BufferedWriter; import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; import java.io.StringWriter; import java.io.Writer; import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; import java.util.Date; import java.util.Map; import java.util.Map.Entry; -import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.apache.solr.common.cloud.Slice; @@ -119,25 +114,24 @@ private static void writeVal(Writer writer, String name, Object v, String update v = Base64.byteArrayToBase64(bytes.array(), bytes.position(),bytes.limit() - bytes.position()); } + XML.Writable valWriter = null; + if(v instanceof SolrInputDocument) { + final SolrInputDocument solrDoc = (SolrInputDocument) v; + valWriter = (writer1) -> solrInputDocumentXmlOutput(solrDoc, writer1); + } else if(v != null) { + final Object val = v; + valWriter = (writer1) -> XML.escapeCharData(val.toString(), writer1); + } + if (update == null) { if (v != null) { - if(v instanceof SolrInputDocument) { - final SolrInputDocument currChildDoc = (SolrInputDocument) v; - XML.writeUnescapedXML(writer, "field", (writer1) -> solrInputDocumentXmlOutput(currChildDoc, writer1), "name", name); - } else { - XML.writeXML(writer, "field", v.toString(), "name", name ); - } + XML.writeUnescapedXML(writer, "field", valWriter, "name", name); } } else { if (v == null) { XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true); } else { - if(v instanceof SolrInputDocument) { - final SolrInputDocument currChildDoc = (SolrInputDocument) v; - XML.writeUnescapedXML(writer, "field", (writer1) -> solrInputDocumentXmlOutput(currChildDoc, writer1), "name", name); - } else { - XML.writeXML(writer, "field", v.toString(), "name", name, "update", update); - } + XML.writeUnescapedXML(writer, "field", valWriter, "name", name, "update", update); } } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/XML.java b/solr/solrj/src/java/org/apache/solr/common/util/XML.java index f7d6773ea1da..ac47bf68944e 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/XML.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/XML.java @@ -180,7 +180,7 @@ public static void writeXML(Writer out, String tag, String val, Map Date: Mon, 6 Aug 2018 14:12:46 +0300 Subject: [PATCH 07/13] SOLR-12485: remove uneeded method ClientUtils#solrInputDocumentXmlOutput --- .../org/apache/solr/client/solrj/util/ClientUtils.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index 5dfaf8144a96..5caaec6c4038 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -117,7 +117,7 @@ private static void writeVal(Writer writer, String name, Object v, String update XML.Writable valWriter = null; if(v instanceof SolrInputDocument) { final SolrInputDocument solrDoc = (SolrInputDocument) v; - valWriter = (writer1) -> solrInputDocumentXmlOutput(solrDoc, writer1); + valWriter = (writer1) -> writeXML(solrDoc, writer1); } else if(v != null) { final Object val = v; valWriter = (writer1) -> XML.escapeCharData(val.toString(), writer1); @@ -136,11 +136,6 @@ private static void writeVal(Writer writer, String name, Object v, String update } } - private static void solrInputDocumentXmlOutput(SolrInputDocument v, Writer writer) throws IOException { - writeXML(v, writer); - } - - public static String toXML( SolrInputDocument doc ) { StringWriter str = new StringWriter(); From 1f39be610a6f31d924ce87e5e79ac54be6a598c8 Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 6 Aug 2018 16:35:01 +0300 Subject: [PATCH 08/13] SOLR-12485: refactor XML utils to use lambdas --- .../solr/client/solrj/util/ClientUtils.java | 4 +- .../java/org/apache/solr/common/util/XML.java | 81 +++++++------------ 2 files changed, 31 insertions(+), 54 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index 5caaec6c4038..ca29a6f81566 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -125,13 +125,13 @@ private static void writeVal(Writer writer, String name, Object v, String update if (update == null) { if (v != null) { - XML.writeUnescapedXML(writer, "field", valWriter, "name", name); + XML.writeUnescapedAttrXML(writer, "field", valWriter, "name", name); } } else { if (v == null) { XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true); } else { - XML.writeUnescapedXML(writer, "field", valWriter, "name", name, "update", update); + XML.writeUnescapedAttrXML(writer, "field", valWriter, "name", name, "update", update); } } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/XML.java b/solr/solrj/src/java/org/apache/solr/common/util/XML.java index ac47bf68944e..ad459fa6bac0 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/XML.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/XML.java @@ -19,6 +19,7 @@ import java.io.Writer; import java.io.IOException; import java.util.Map; +import java.util.stream.Stream; /** * @@ -101,12 +102,18 @@ public final static void writeXML(Writer out, String tag, String val) throws IOE } } + /** does NOT escape character data in val or attributes, must already be valid XML */ public final static void writeUnescapedXML(Writer out, String tag, String val, Object... attrs) throws IOException { - writeUnescapedXML(out, tag, (writer1) -> writer1.write(val), attrs); + writeUnescapedAttrXML(out, tag, (writer1) -> writer1.write(val), attrs); } - /** does NOT escape character data in val, must already be valid XML */ - public final static void writeUnescapedXML(Writer out, String tag, Writable val, Object... attrs) throws IOException { + /** does NOT escape character data in attributes, must already be valid XML */ + public final static void writeUnescapedAttrXML(Writer out, String tag, Writable valWritable, Object... attrs) throws IOException { + writeXML(out, tag, valWritable, null, attrs); + } + + private final static void writeXML(Writer out, String tag, Writable valWritable, + Escapable attrWriteFunc, Object... attrs) throws IOException { out.write('<'); out.write(tag); for (int i=0; i'); } else { out.write('>'); - val.write(out); + valWritable.write(out); out.write('<'); out.write('/'); out.write(tag); @@ -130,54 +140,15 @@ public final static void writeUnescapedXML(Writer out, String tag, Writable val, } } - /** escapes character data in val */ + /** escapes character data in val and attributes */ public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException { - out.write('<'); - out.write(tag); - for (int i=0; i'); - } else { - out.write('>'); - escapeCharData(val,out); - out.write('<'); - out.write('/'); - out.write(tag); - out.write('>'); - } + final Writable writable = val!= null? (writer1) -> XML.escapeCharData(val, writer1): null; + writeXML(out, tag, writable, XML::escapeAttributeValue, attrs); } - /** escapes character data in val */ + /** escapes character data in val and attributes */ public static void writeXML(Writer out, String tag, String val, Map attrs) throws IOException { - out.write('<'); - out.write(tag); - for (Map.Entry entry : attrs.entrySet()) { - out.write(' '); - out.write(entry.getKey()); - out.write('='); - out.write('"'); - escapeAttributeValue(entry.getValue(), out); - out.write('"'); - } - if (val == null) { - out.write('/'); - out.write('>'); - } else { - out.write('>'); - escapeCharData(val,out); - out.write('<'); - out.write('/'); - out.write(tag); - out.write('>'); - } + writeXML(out, tag, val, attrs.entrySet().stream().flatMap((entry) -> Stream.of(entry.getKey(), entry.getValue())).toArray()); } public static void escape(char [] chars, int offset, int length, Writer out, String [] escapes) throws IOException{ @@ -194,7 +165,7 @@ public static void escape(char [] chars, int offset, int length, Writer out, Str } } - public static void escape(String str, Writer out, String[] escapes) throws IOException { + private static void escape(String str, Writer out, String[] escapes) throws IOException { for (int i=0; i { + void escape(T val, U out) throws IOException; } } From 06039d5f722d431427dc28849f07bec49f2f6f17 Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 6 Aug 2018 16:37:01 +0300 Subject: [PATCH 09/13] SOLR-12485: add comments to explain use of isLabeledChildDoc --- .../src/java/org/apache/solr/handler/loader/XMLLoader.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java index 825fc73922cd..724a40c129bf 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java @@ -455,9 +455,10 @@ public SolrInputDocument readDoc(XMLStreamReader parser) throws XMLStreamExcepti break; } if(!isLabeledChildDoc){ - // do not add empty data if a labeled child document was added + // only add data if this is not a childDoc, since it was added already doc.addField(name, v); } else { + // reset so next field is not treated as child doc isLabeledChildDoc = false; } // field is over From dd51c83f816c77b0f0127a84db9acf3830c0edc6 Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 6 Aug 2018 17:18:45 +0300 Subject: [PATCH 10/13] SOLR-12485: rename XML#writeUnescapedAttrXML to XML#writeElementXML --- .../java/org/apache/solr/client/solrj/util/ClientUtils.java | 4 ++-- solr/solrj/src/java/org/apache/solr/common/util/XML.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index ca29a6f81566..2d697da392aa 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -125,13 +125,13 @@ private static void writeVal(Writer writer, String name, Object v, String update if (update == null) { if (v != null) { - XML.writeUnescapedAttrXML(writer, "field", valWriter, "name", name); + XML.writeElementXML(writer, "field", valWriter, "name", name); } } else { if (v == null) { XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true); } else { - XML.writeUnescapedAttrXML(writer, "field", valWriter, "name", name, "update", update); + XML.writeElementXML(writer, "field", valWriter, "name", name, "update", update); } } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/XML.java b/solr/solrj/src/java/org/apache/solr/common/util/XML.java index ad459fa6bac0..c74945193a84 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/XML.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/XML.java @@ -104,11 +104,11 @@ public final static void writeXML(Writer out, String tag, String val) throws IOE /** does NOT escape character data in val or attributes, must already be valid XML */ public final static void writeUnescapedXML(Writer out, String tag, String val, Object... attrs) throws IOException { - writeUnescapedAttrXML(out, tag, (writer1) -> writer1.write(val), attrs); + writeElementXML(out, tag, (writer1) -> writer1.write(val), attrs); } /** does NOT escape character data in attributes, must already be valid XML */ - public final static void writeUnescapedAttrXML(Writer out, String tag, Writable valWritable, Object... attrs) throws IOException { + public final static void writeElementXML(Writer out, String tag, Writable valWritable, Object... attrs) throws IOException { writeXML(out, tag, valWritable, null, attrs); } From 498f7888a752932b9975cac276a43db9cefc03ec Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 7 Aug 2018 11:32:44 +0300 Subject: [PATCH 11/13] SOLR-12485: remove XML#writeElementXML --- .../solr/client/solrj/util/ClientUtils.java | 6 ++--- .../java/org/apache/solr/common/util/XML.java | 24 ++++--------------- .../org/apache/solr/util/BaseTestHarness.java | 2 +- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index 2d697da392aa..d329c7a4b588 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -125,13 +125,13 @@ private static void writeVal(Writer writer, String name, Object v, String update if (update == null) { if (v != null) { - XML.writeElementXML(writer, "field", valWriter, "name", name); + XML.writeXML(writer, "field", valWriter, "name", name); } } else { if (v == null) { - XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true); + XML.writeXML(writer, "field", (String) null, "name", name, "update", update, "null", true); } else { - XML.writeElementXML(writer, "field", valWriter, "name", name, "update", update); + XML.writeXML(writer, "field", valWriter, "name", name, "update", update); } } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/XML.java b/solr/solrj/src/java/org/apache/solr/common/util/XML.java index c74945193a84..a391a98e1b04 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/XML.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/XML.java @@ -104,16 +104,10 @@ public final static void writeXML(Writer out, String tag, String val) throws IOE /** does NOT escape character data in val or attributes, must already be valid XML */ public final static void writeUnescapedXML(Writer out, String tag, String val, Object... attrs) throws IOException { - writeElementXML(out, tag, (writer1) -> writer1.write(val), attrs); + writeXML(out, tag, (writer1) -> writer1.write(val), attrs); } - /** does NOT escape character data in attributes, must already be valid XML */ - public final static void writeElementXML(Writer out, String tag, Writable valWritable, Object... attrs) throws IOException { - writeXML(out, tag, valWritable, null, attrs); - } - - private final static void writeXML(Writer out, String tag, Writable valWritable, - Escapable attrWriteFunc, Object... attrs) throws IOException { + public final static void writeXML(Writer out, String tag, Writable valWritable, Object... attrs) throws IOException { out.write('<'); out.write(tag); for (int i=0; i XML.escapeCharData(val, writer1): null; - writeXML(out, tag, writable, XML::escapeAttributeValue, attrs); + writeXML(out, tag, writable, attrs); } /** escapes character data in val and attributes */ @@ -151,7 +142,7 @@ public static void writeXML(Writer out, String tag, String val, Map Stream.of(entry.getKey(), entry.getValue())).toArray()); } - public static void escape(char [] chars, int offset, int length, Writer out, String [] escapes) throws IOException{ + private static void escape(char [] chars, int offset, int length, Writer out, String [] escapes) throws IOException{ for (int i=offset; i { - void escape(T val, U out) throws IOException; - } } diff --git a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java index a84d6d17dc26..6c77fd921776 100644 --- a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java +++ b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java @@ -206,7 +206,7 @@ public static String simpleTag(String tag, String... args) { if (null == args || 0 == args.length) { XML.writeXML(r, tag, null); } else { - XML.writeXML(r, tag, null, (Object[])args); + XML.writeXML(r, tag, (String) null, (Object[])args); } return r.getBuffer().toString(); } catch (IOException e) { From 671a63651231fd0c9c013ddc074632f12124ea1c Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 7 Aug 2018 11:35:54 +0300 Subject: [PATCH 12/13] SOLR-12485: remove XML#writeElementXML --- .../src/java/org/apache/solr/client/solrj/util/ClientUtils.java | 2 +- .../src/java/org/apache/solr/util/BaseTestHarness.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index d329c7a4b588..26a188d67b2c 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -129,7 +129,7 @@ private static void writeVal(Writer writer, String name, Object v, String update } } else { if (v == null) { - XML.writeXML(writer, "field", (String) null, "name", name, "update", update, "null", true); + XML.writeXML(writer, "field", (XML.Writable) null, "name", name, "update", update, "null", true); } else { XML.writeXML(writer, "field", valWriter, "name", name, "update", update); } diff --git a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java index 6c77fd921776..e310f9b590e9 100644 --- a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java +++ b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java @@ -206,7 +206,7 @@ public static String simpleTag(String tag, String... args) { if (null == args || 0 == args.length) { XML.writeXML(r, tag, null); } else { - XML.writeXML(r, tag, (String) null, (Object[])args); + XML.writeXML(r, tag, (XML.Writable) null, (Object[])args); } return r.getBuffer().toString(); } catch (IOException e) { From efc39c68777ce205004f01c871ee45e95389dcc4 Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 7 Aug 2018 17:35:15 +0300 Subject: [PATCH 13/13] SOLR-12485: change back to (String) null --- .../src/java/org/apache/solr/client/solrj/util/ClientUtils.java | 2 +- .../src/java/org/apache/solr/util/BaseTestHarness.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java index 26a188d67b2c..d329c7a4b588 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java @@ -129,7 +129,7 @@ private static void writeVal(Writer writer, String name, Object v, String update } } else { if (v == null) { - XML.writeXML(writer, "field", (XML.Writable) null, "name", name, "update", update, "null", true); + XML.writeXML(writer, "field", (String) null, "name", name, "update", update, "null", true); } else { XML.writeXML(writer, "field", valWriter, "name", name, "update", update); } diff --git a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java index e310f9b590e9..6c77fd921776 100644 --- a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java +++ b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java @@ -206,7 +206,7 @@ public static String simpleTag(String tag, String... args) { if (null == args || 0 == args.length) { XML.writeXML(r, tag, null); } else { - XML.writeXML(r, tag, (XML.Writable) null, (Object[])args); + XML.writeXML(r, tag, (String) null, (Object[])args); } return r.getBuffer().toString(); } catch (IOException e) {