From 6da7b4b2447f2dd20dbca031974e79f69189f953 Mon Sep 17 00:00:00 2001 From: user Date: Tue, 5 Jun 2018 11:58:10 +0300 Subject: [PATCH 01/34] first tests --- .../org/apache/solr/schema/IndexSchema.java | 3 + .../DeeplyNestedUpdateProcessor.java | 70 ++++++++++++ .../DeeplyNestedUpdateProcessorFactory.java | 63 +++++++++++ .../solr/collection1/conf/schema15.xml | 4 + .../solrconfig-update-processor-chains.xml | 6 + .../TestDeeplyNestedUpdateProcessor.java | 105 ++++++++++++++++++ 6 files changed, 251 insertions(+) create mode 100644 solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java create mode 100644 solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java create mode 100644 solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index 7f86f27afffc..2c63af47f261 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -104,9 +104,12 @@ public class IndexSchema { public static final String FIELD_TYPE = "fieldType"; public static final String FIELD_TYPES = FIELD_TYPE + "s"; public static final String INTERNAL_POLY_FIELD_PREFIX = "*" + FieldType.POLY_FIELD_SEPARATOR; + public static final String LEVEL_FIELD_NAME = "_nestLevel_"; public static final String LUCENE_MATCH_VERSION_PARAM = "luceneMatchVersion"; public static final String MAX_CHARS = "maxChars"; public static final String NAME = "name"; + public static final String PARENT_FIELD_NAME = "_nestParent_"; + public static final String PATH_FIELD_NAME = "_nestPath_"; public static final String REQUIRED = "required"; public static final String SCHEMA = "schema"; public static final String SIMILARITY = "similarity"; diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java new file mode 100644 index 000000000000..014eedb3ed55 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.update.AddUpdateCommand; + +public class DeeplyNestedUpdateProcessor extends UpdateRequestProcessor { + String[] fields; + + protected DeeplyNestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, List fields, UpdateRequestProcessor next) { + super(next); + this.fields = fields.toArray(new String[0]); + } + + @Override + public void processAdd(AddUpdateCommand cmd) throws IOException { + SolrInputDocument doc = cmd.getSolrInputDocument(); + processDocChildren(doc, null); + } + + private void processDocChildren(SolrInputDocument doc, String fullPath) { + for(SolrInputField field: doc.values()) { + if(field.getFirstValue() instanceof SolrInputDocument) { + Object val = field.getValue(); + fullPath = String.format("%s%s", Objects.toString(fullPath + ".", ""), field.getName()); + if (val instanceof Collection) { + for(Object childDoc: (Collection) val) { + processChildDoc((SolrInputDocument) childDoc, doc, fullPath); + } + } else { + processChildDoc((SolrInputDocument) val, doc, fullPath); + } + } + } + } + + private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String fullPath) { + sdoc.addField(IndexSchema.PATH_FIELD_NAME, fullPath); + sdoc.addField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue("id")); + sdoc.addField(IndexSchema.LEVEL_FIELD_NAME, fullPath.split(".").length +1); + } + +} diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java new file mode 100644 index 000000000000..e91450604044 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update.processor; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.StrUtils; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; + +import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; + +public class DeeplyNestedUpdateProcessorFactory extends UpdateRequestProcessorFactory { + + private static List allowedFields = new ArrayList(3) { + { + add(IndexSchema.LEVEL_FIELD_NAME);add(IndexSchema.PARENT_FIELD_NAME); + add(IndexSchema.PATH_FIELD_NAME); + } + }; + private List fields; + + public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next ) { + return new DeeplyNestedUpdateProcessor(req, rsp, fields, next); + } + + @Override + public void init( NamedList args ) + { + Object tmp = args.remove("fields"); + if (null == tmp) { + throw new SolrException(SERVER_ERROR, + "'versionField' must be configured"); + } + if (! (tmp instanceof String) ) { + throw new SolrException(SERVER_ERROR, + "'versionField' must be configured as a "); + } + fields = StrUtils.splitSmart((String)tmp, ','); + if(!allowedFields.containsAll(fields)) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Deeply Nested Fields may only contain _nestLevel_, _nestParent_, _nestPath_"); + } + } +} diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml index cae5aeea9c11..dd565879f6ac 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml @@ -568,6 +568,10 @@ + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml index 4113bd133eb5..235c774c82f3 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml @@ -30,6 +30,12 @@ + + + _nestLevel_,_nestParent_,_nestPath_ + + + diff --git a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java new file mode 100644 index 000000000000..67efa9ed02ae --- /dev/null +++ b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import java.io.IOException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.ContentStream; +import org.apache.solr.common.util.ContentStreamBase; +import org.apache.solr.handler.UpdateRequestHandler; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.SolrQueryParser; +import org.apache.solr.servlet.DirectSolrConnection; +import org.apache.solr.servlet.SolrRequestParsers; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestDeeplyNestedUpdateProcessor extends SolrTestCaseJ4 { + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-update-processor-chains.xml", "schema15.xml"); + } + + @Before + public void before() throws Exception { + assertU(delQ("*:*")); + assertU(commit()); + } + + @Test + public void testDeeplyNestedURP() throws Exception { + SolrInputDocument document1 = sdoc("id", 1, "parent_s", "X", + "child1_s", sdoc("id", 2, "child_s", "y"), + "child2_s", sdoc("id", 3, "child_s", "z")); + + List streams = new ArrayList<>( 1 ); + final String xmlDoc = ClientUtils.toXML(document1); + if( xmlDoc.length() > 1 ) { + streams.add( new ContentStreamBase.StringStream( xmlDoc ) ); + } + + SolrQueryRequest req; + try { + req = new SolrRequestParsers(h.getCore().getSolrConfig()).buildRequestFrom( h.getCore(), params("update.chain", "deeply-nested"), streams ); + SolrQueryResponse rsp = new SolrQueryResponse(); + SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); + h.getCore().execute( h.getCore().getRequestHandler("/update"), req, rsp ); + if( rsp.getException() != null ) { + throw rsp.getException(); + } + } catch (Exception e) { + throw e; + } + + indexDeeplyNestedSolrInputDocumentsDirectly(document1); + } + + private long getNewClock() { + long time = System.currentTimeMillis(); + return time << 20; + } + + private void indexDeeplyNestedSolrInputDocumentsDirectly(SolrInputDocument... docs) throws IOException { +// final String value = "Kittens!!! \u20AC"; +// final String reqDoc = "42"+value+""; +// .request( "/update?"+CommonParams.STREAM_BODY+"="+URLEncoder.encode(reqDoc, "UTF-8"), null ); +// h.getCore().execute(new UpdateRequestHandler(), , new SolrQueryResponse()); + SolrQueryRequest coreReq = new LocalSolrQueryRequest(h.getCore(), params("update.chain", "deeply-nested")); + AddUpdateCommand updateCmd = new AddUpdateCommand(coreReq); + for (SolrInputDocument doc: docs) { + long version = getNewClock(); + updateCmd.setVersion(Math.abs(version)); + updateCmd.solrDoc = doc; + h.getCore().getUpdateHandler().addDoc(updateCmd); + updateCmd.clear(); + } + assertU(commit()); + } +} From 6ba4debda4e77198330a5e157c85564cdd8981dd Mon Sep 17 00:00:00 2001 From: user Date: Wed, 6 Jun 2018 03:31:47 +0300 Subject: [PATCH 02/34] only index fields from conf --- .../DeeplyNestedUpdateProcessor.java | 18 +++--- .../TestDeeplyNestedUpdateProcessor.java | 55 ++++++++----------- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java index 014eedb3ed55..64c3de086ff9 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java @@ -32,27 +32,30 @@ import org.apache.solr.update.AddUpdateCommand; public class DeeplyNestedUpdateProcessor extends UpdateRequestProcessor { - String[] fields; + List fields; protected DeeplyNestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, List fields, UpdateRequestProcessor next) { super(next); - this.fields = fields.toArray(new String[0]); + this.fields = fields; } @Override public void processAdd(AddUpdateCommand cmd) throws IOException { SolrInputDocument doc = cmd.getSolrInputDocument(); processDocChildren(doc, null); + return; } private void processDocChildren(SolrInputDocument doc, String fullPath) { for(SolrInputField field: doc.values()) { if(field.getFirstValue() instanceof SolrInputDocument) { Object val = field.getValue(); - fullPath = String.format("%s%s", Objects.toString(fullPath + ".", ""), field.getName()); + fullPath = Objects.isNull(fullPath) ? field.getName(): String.format("%s.%s", fullPath, field.getName()); if (val instanceof Collection) { for(Object childDoc: (Collection) val) { - processChildDoc((SolrInputDocument) childDoc, doc, fullPath); + if(val instanceof SolrInputDocument) { + processChildDoc((SolrInputDocument) childDoc, doc, fullPath); + } } } else { processChildDoc((SolrInputDocument) val, doc, fullPath); @@ -62,9 +65,10 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { } private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String fullPath) { - sdoc.addField(IndexSchema.PATH_FIELD_NAME, fullPath); - sdoc.addField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue("id")); - sdoc.addField(IndexSchema.LEVEL_FIELD_NAME, fullPath.split(".").length +1); + if(fields.contains(IndexSchema.PATH_FIELD_NAME)) sdoc.addField(IndexSchema.PATH_FIELD_NAME, fullPath); + if(fields.contains(IndexSchema.PARENT_FIELD_NAME)) sdoc.addField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue("id")); + if(fields.contains(IndexSchema.LEVEL_FIELD_NAME)) sdoc.addField(IndexSchema.LEVEL_FIELD_NAME, fullPath.split("\\.").length); + processDocChildren(sdoc, fullPath); } } diff --git a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java index 67efa9ed02ae..007ebacdd177 100644 --- a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java @@ -55,15 +55,31 @@ public void before() throws Exception { @Test public void testDeeplyNestedURP() throws Exception { - SolrInputDocument document1 = sdoc("id", 1, "parent_s", "X", - "child1_s", sdoc("id", 2, "child_s", "y"), - "child2_s", sdoc("id", 3, "child_s", "z")); + final String jDoc = "{\n" + + " \"add\": {\n" + + " \"doc\": {\n" + + " \"id\": \"1\",\n" + + " \"children\": [\n" + + " {\n" + + " \"id\": \"2\",\n" + + " \"foo_s\": \"Yaz\"\n" + + " \"grandChild\": \n" + + " {\n" + + " \"id\": \"4\",\n" + + " \"foo_s\": \"Jazz\"\n" + + " },\n" + + " },\n" + + " {\n" + + " \"id\": \"3\",\n" + + " \"foo_s\": \"Bar\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; List streams = new ArrayList<>( 1 ); - final String xmlDoc = ClientUtils.toXML(document1); - if( xmlDoc.length() > 1 ) { - streams.add( new ContentStreamBase.StringStream( xmlDoc ) ); - } + streams.add( new ContentStreamBase.StringStream( jDoc ) ); SolrQueryRequest req; try { @@ -77,29 +93,6 @@ public void testDeeplyNestedURP() throws Exception { } catch (Exception e) { throw e; } - - indexDeeplyNestedSolrInputDocumentsDirectly(document1); - } - - private long getNewClock() { - long time = System.currentTimeMillis(); - return time << 20; - } - - private void indexDeeplyNestedSolrInputDocumentsDirectly(SolrInputDocument... docs) throws IOException { -// final String value = "Kittens!!! \u20AC"; -// final String reqDoc = "42"+value+""; -// .request( "/update?"+CommonParams.STREAM_BODY+"="+URLEncoder.encode(reqDoc, "UTF-8"), null ); -// h.getCore().execute(new UpdateRequestHandler(), , new SolrQueryResponse()); - SolrQueryRequest coreReq = new LocalSolrQueryRequest(h.getCore(), params("update.chain", "deeply-nested")); - AddUpdateCommand updateCmd = new AddUpdateCommand(coreReq); - for (SolrInputDocument doc: docs) { - long version = getNewClock(); - updateCmd.setVersion(Math.abs(version)); - updateCmd.solrDoc = doc; - h.getCore().getUpdateHandler().addDoc(updateCmd); - updateCmd.clear(); - } - assertU(commit()); + System.out.println(); } } From 15deec89f7b6633b4655aecc2e061964b4ffce84 Mon Sep 17 00:00:00 2001 From: Moshe Date: Wed, 6 Jun 2018 16:36:31 +0300 Subject: [PATCH 03/34] SOLR-12441: tests with query --- .../DeeplyNestedUpdateProcessor.java | 4 +-- .../solrconfig-update-processor-chains.xml | 1 + .../TestDeeplyNestedUpdateProcessor.java | 32 ++++++++++++++++--- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java index 64c3de086ff9..480ac5f702ff 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java @@ -43,7 +43,7 @@ protected DeeplyNestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rs public void processAdd(AddUpdateCommand cmd) throws IOException { SolrInputDocument doc = cmd.getSolrInputDocument(); processDocChildren(doc, null); - return; + super.processAdd(cmd); } private void processDocChildren(SolrInputDocument doc, String fullPath) { @@ -53,7 +53,7 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { fullPath = Objects.isNull(fullPath) ? field.getName(): String.format("%s.%s", fullPath, field.getName()); if (val instanceof Collection) { for(Object childDoc: (Collection) val) { - if(val instanceof SolrInputDocument) { + if(childDoc instanceof SolrInputDocument) { processChildDoc((SolrInputDocument) childDoc, doc, fullPath); } } diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml index 235c774c82f3..b1fc974d74a2 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml @@ -34,6 +34,7 @@ _nestLevel_,_nestParent_,_nestPath_ + diff --git a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java index 007ebacdd177..c55f004eb4b2 100644 --- a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java @@ -20,20 +20,27 @@ import java.io.IOException; import java.net.URLEncoder; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.MultiMapSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.UpdateParams; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.handler.UpdateRequestHandler; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryRequestBase; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.SolrQueryParser; import org.apache.solr.servlet.DirectSolrConnection; import org.apache.solr.servlet.SolrRequestParsers; @@ -55,6 +62,7 @@ public void before() throws Exception { @Test public void testDeeplyNestedURP() throws Exception { + final String grandChildId = "4"; final String jDoc = "{\n" + " \"add\": {\n" + " \"doc\": {\n" + @@ -65,7 +73,7 @@ public void testDeeplyNestedURP() throws Exception { " \"foo_s\": \"Yaz\"\n" + " \"grandChild\": \n" + " {\n" + - " \"id\": \"4\",\n" + + " \"id\": \""+ grandChildId + "\",\n" + " \"foo_s\": \"Jazz\"\n" + " },\n" + " },\n" + @@ -81,7 +89,7 @@ public void testDeeplyNestedURP() throws Exception { List streams = new ArrayList<>( 1 ); streams.add( new ContentStreamBase.StringStream( jDoc ) ); - SolrQueryRequest req; + SolrQueryRequest req = null; try { req = new SolrRequestParsers(h.getCore().getSolrConfig()).buildRequestFrom( h.getCore(), params("update.chain", "deeply-nested"), streams ); SolrQueryResponse rsp = new SolrQueryResponse(); @@ -90,9 +98,23 @@ public void testDeeplyNestedURP() throws Exception { if( rsp.getException() != null ) { throw rsp.getException(); } - } catch (Exception e) { - throw e; + } finally { + if (req != null) { + req.close(); + SolrRequestInfo.clearRequestInfo(); + } } - System.out.println(); + assertU(commit()); + assertJQ(req("q", IndexSchema.LEVEL_FIELD_NAME + ":2", + "fl","*", + "sort","id desc", + "wt","json"), + "/response/docs/[0]/id=='" + grandChildId + "'"); + + assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*.grandChild", + "fl","*", + "sort","id desc", + "wt","json"), + "/response/docs/[0]/id=='" + grandChildId + "'"); } } From 76dd0d7155c94b8c6397032a25cf933abe4579e2 Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 11 Jun 2018 10:54:56 +0300 Subject: [PATCH 04/34] SOLR-12441: use EnumSet for conf --- .../DeeplyNestedUpdateProcessor.java | 47 +++++++++++++++---- .../DeeplyNestedUpdateProcessorFactory.java | 35 +++++++++----- .../solrconfig-update-processor-chains.xml | 2 +- .../TestDeeplyNestedUpdateProcessor.java | 16 ------- 4 files changed, 61 insertions(+), 39 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java index 480ac5f702ff..e9e98ea59d89 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java @@ -18,24 +18,27 @@ package org.apache.solr.update.processor; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; +import java.util.EnumSet; +import java.util.Locale; import java.util.Objects; -import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.IndexSchema; import org.apache.solr.update.AddUpdateCommand; +import static org.apache.solr.update.processor.DeeplyNestedUpdateProcessorFactory.NestedFlag; public class DeeplyNestedUpdateProcessor extends UpdateRequestProcessor { - List fields; + private EnumSet fields; + SolrQueryRequest req; - protected DeeplyNestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, List fields, UpdateRequestProcessor next) { + + protected DeeplyNestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, EnumSet fields, UpdateRequestProcessor next) { super(next); + this.req = req; this.fields = fields; } @@ -50,7 +53,7 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { for(SolrInputField field: doc.values()) { if(field.getFirstValue() instanceof SolrInputDocument) { Object val = field.getValue(); - fullPath = Objects.isNull(fullPath) ? field.getName(): String.format("%s.%s", fullPath, field.getName()); + fullPath = Objects.isNull(fullPath) ? field.getName(): String.format(Locale.ROOT,"%s.%s", fullPath, field.getName()); if (val instanceof Collection) { for(Object childDoc: (Collection) val) { if(childDoc instanceof SolrInputDocument) { @@ -65,10 +68,34 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { } private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String fullPath) { - if(fields.contains(IndexSchema.PATH_FIELD_NAME)) sdoc.addField(IndexSchema.PATH_FIELD_NAME, fullPath); - if(fields.contains(IndexSchema.PARENT_FIELD_NAME)) sdoc.addField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue("id")); - if(fields.contains(IndexSchema.LEVEL_FIELD_NAME)) sdoc.addField(IndexSchema.LEVEL_FIELD_NAME, fullPath.split("\\.").length); + if(fields == NestedFlag.ALL) { + setPathField(sdoc, fullPath); + setParentKey(sdoc, parent); + setLevelKey(sdoc, fullPath); + } else { + if(fields.contains(NestedFlag.PATH)) { + setPathField(sdoc, fullPath); + } + if (fields.contains(NestedFlag.PARENT)) { + setParentKey(sdoc, parent); + } + if(fields.contains(NestedFlag.LEVEL)) { + setLevelKey(sdoc, fullPath); + } + } processDocChildren(sdoc, fullPath); } -} + private void setLevelKey(SolrInputDocument sdoc, String fullPath) { + sdoc.addField(IndexSchema.LEVEL_FIELD_NAME, fullPath.split("\\.").length); + } + + private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { + sdoc.addField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue(req.getSchema().getUniqueKeyField().getName())); + } + + private void setPathField(SolrInputDocument sdoc, String fullPath) { + sdoc.addField(IndexSchema.PATH_FIELD_NAME, fullPath); + } + +} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java index e91450604044..c63a01246da7 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java @@ -17,27 +17,24 @@ package org.apache.solr.update.processor; -import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; import org.apache.solr.common.SolrException; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.StrUtils; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.schema.IndexSchema; import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; public class DeeplyNestedUpdateProcessorFactory extends UpdateRequestProcessorFactory { - private static List allowedFields = new ArrayList(3) { - { - add(IndexSchema.LEVEL_FIELD_NAME);add(IndexSchema.PARENT_FIELD_NAME); - add(IndexSchema.PATH_FIELD_NAME); - } - }; - private List fields; + private EnumSet fields; + private static final List allowedConfFields = NestedFlag.ALL.stream().map(e -> e.toString().toLowerCase(Locale.ROOT)).collect(Collectors.toList()); public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next ) { return new DeeplyNestedUpdateProcessor(req, rsp, fields, next); @@ -55,9 +52,23 @@ public void init( NamedList args ) throw new SolrException(SERVER_ERROR, "'versionField' must be configured as a "); } - fields = StrUtils.splitSmart((String)tmp, ','); - if(!allowedFields.containsAll(fields)) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Deeply Nested Fields may only contain _nestLevel_, _nestParent_, _nestPath_"); + List fields = StrUtils.splitSmart((String)tmp, ','); + if(!allowedConfFields.containsAll(fields)) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Deeply Nested URP may only contain: " + StringUtils.join(allowedConfFields, ", ") + + " got: " + StringUtils.join(fields, ", ") + " instead"); } + this.fields = fields.size() == NestedFlag.values().length ? NestedFlag.ALL: generateNestedFlags(fields); + } + + private static EnumSet generateNestedFlags(List fields) { + return EnumSet.copyOf(fields.stream().map(e -> NestedFlag.valueOf(e.toUpperCase(Locale.ROOT))).collect(Collectors.toList())); + } + + public enum NestedFlag { + PATH, + PARENT, + LEVEL; + + public static final EnumSet ALL = EnumSet.allOf(NestedFlag.class); } } diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml index b1fc974d74a2..bcce74c4d9af 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml @@ -32,7 +32,7 @@ - _nestLevel_,_nestParent_,_nestPath_ + level,parent,path diff --git a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java index c55f004eb4b2..537de6d58374 100644 --- a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java @@ -17,32 +17,16 @@ package org.apache.solr.update; -import java.io.IOException; -import java.net.URLEncoder; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.MultiMapSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.params.UpdateParams; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStreamBase; -import org.apache.solr.handler.UpdateRequestHandler; -import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrQueryRequestBase; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.IndexSchema; -import org.apache.solr.search.SolrQueryParser; -import org.apache.solr.servlet.DirectSolrConnection; import org.apache.solr.servlet.SolrRequestParsers; import org.junit.Before; import org.junit.BeforeClass; From 6196945d16ec095fe3cc45d47b94450a062c129d Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 25 Jun 2018 12:03:17 +0300 Subject: [PATCH 05/34] SOLR-12441: add tests for children too --- .../TestDeeplyNestedUpdateProcessor.java | 50 ++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java index 537de6d58374..559b9f2e81b0 100644 --- a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java @@ -33,6 +33,10 @@ import org.junit.Test; public class TestDeeplyNestedUpdateProcessor extends SolrTestCaseJ4 { + + public static final String[] childrenIds = { "2", "3" }; + public static final String grandChildId = "4"; + @BeforeClass public static void beforeClass() throws Exception { initCore("solrconfig-update-processor-chains.xml", "schema15.xml"); @@ -45,8 +49,39 @@ public void before() throws Exception { } @Test - public void testDeeplyNestedURP() throws Exception { - final String grandChildId = "4"; + public void testDeeplyNestedURPGrandChild() throws Exception { + indexSampleData(); + assertJQ(req("q", IndexSchema.LEVEL_FIELD_NAME + ":2", + "fl","*", + "sort","id desc", + "wt","json"), + "/response/docs/[0]/id=='" + grandChildId + "'"); + + assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*.grandChild", + "fl","*", + "sort","id desc", + "wt","json"), + "/response/docs/[0]/id=='" + grandChildId + "'"); + } + + @Test + public void testDeeplyNestedURPChildren() throws Exception { + final String[] childrenTests = {"/response/docs/[0]/id=='" + childrenIds[0] + "'", "/response/docs/[1]/id=='" + childrenIds[1] + "'"}; + indexSampleData(); + assertJQ(req("q", IndexSchema.LEVEL_FIELD_NAME + ":1", + "fl","*", + "sort","id asc", + "wt","json"), + childrenTests); + + assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":children", + "fl","*", + "sort","id asc", + "wt","json"), + childrenTests); + } + + private void indexSampleData() throws Exception { final String jDoc = "{\n" + " \"add\": {\n" + " \"doc\": {\n" + @@ -89,16 +124,5 @@ public void testDeeplyNestedURP() throws Exception { } } assertU(commit()); - assertJQ(req("q", IndexSchema.LEVEL_FIELD_NAME + ":2", - "fl","*", - "sort","id desc", - "wt","json"), - "/response/docs/[0]/id=='" + grandChildId + "'"); - - assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*.grandChild", - "fl","*", - "sort","id desc", - "wt","json"), - "/response/docs/[0]/id=='" + grandChildId + "'"); } } From 63a8c44735ad3351caec0ea688523c17d018e197 Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 09:16:44 +0300 Subject: [PATCH 06/34] SOLR-12441: prefix nested fields with NEST --- solr/core/src/java/org/apache/solr/schema/IndexSchema.java | 6 +++--- solr/core/src/test-files/solr/collection1/conf/schema15.xml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index 2c63af47f261..4f343e8f9529 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -104,12 +104,12 @@ public class IndexSchema { public static final String FIELD_TYPE = "fieldType"; public static final String FIELD_TYPES = FIELD_TYPE + "s"; public static final String INTERNAL_POLY_FIELD_PREFIX = "*" + FieldType.POLY_FIELD_SEPARATOR; - public static final String LEVEL_FIELD_NAME = "_nestLevel_"; + public static final String LEVEL_FIELD_NAME = "_NEST_LEVEL_"; public static final String LUCENE_MATCH_VERSION_PARAM = "luceneMatchVersion"; public static final String MAX_CHARS = "maxChars"; public static final String NAME = "name"; - public static final String PARENT_FIELD_NAME = "_nestParent_"; - public static final String PATH_FIELD_NAME = "_nestPath_"; + public static final String PARENT_FIELD_NAME = "_NEST_PARENT_"; + public static final String PATH_FIELD_NAME = "_NEST_PATH_"; public static final String REQUIRED = "required"; public static final String SCHEMA = "schema"; public static final String SIMILARITY = "similarity"; diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml index dd565879f6ac..b7c6b39da654 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml @@ -569,9 +569,9 @@ - - - + + + From d7825632ca0747943c8b39b557b0b28fd4cff136 Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 09:29:45 +0300 Subject: [PATCH 07/34] SOLR-12441: use iterator for field values --- .../processor/DeeplyNestedUpdateProcessor.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java index e9e98ea59d89..5a9c88b8a567 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java @@ -51,17 +51,10 @@ public void processAdd(AddUpdateCommand cmd) throws IOException { private void processDocChildren(SolrInputDocument doc, String fullPath) { for(SolrInputField field: doc.values()) { - if(field.getFirstValue() instanceof SolrInputDocument) { - Object val = field.getValue(); - fullPath = Objects.isNull(fullPath) ? field.getName(): String.format(Locale.ROOT,"%s.%s", fullPath, field.getName()); - if (val instanceof Collection) { - for(Object childDoc: (Collection) val) { - if(childDoc instanceof SolrInputDocument) { - processChildDoc((SolrInputDocument) childDoc, doc, fullPath); - } - } - } else { - processChildDoc((SolrInputDocument) val, doc, fullPath); + for(Object val: field) { + if(val instanceof SolrInputDocument) { + final String jointPath = Objects.isNull(fullPath) ? field.getName(): String.format(Locale.ROOT,"%s.%s", fullPath, field.getName()); + processChildDoc((SolrInputDocument) val, doc, jointPath); } } } From 4f4b48ddcfef586c6c7e29f41aef2702f245a97e Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 10:02:58 +0300 Subject: [PATCH 08/34] SOLR-12441: remove NestedFlags.ALL and LEVEL_FIELD_NAME --- .../org/apache/solr/schema/IndexSchema.java | 1 - .../DeeplyNestedUpdateProcessor.java | 29 +++++++------------ .../DeeplyNestedUpdateProcessorFactory.java | 10 +++---- .../solr/collection1/conf/schema15.xml | 1 - .../solrconfig-update-processor-chains.xml | 2 +- .../TestDeeplyNestedUpdateProcessor.java | 10 ------- 6 files changed, 15 insertions(+), 38 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index 4f343e8f9529..f124f0e7089d 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -104,7 +104,6 @@ public class IndexSchema { public static final String FIELD_TYPE = "fieldType"; public static final String FIELD_TYPES = FIELD_TYPE + "s"; public static final String INTERNAL_POLY_FIELD_PREFIX = "*" + FieldType.POLY_FIELD_SEPARATOR; - public static final String LEVEL_FIELD_NAME = "_NEST_LEVEL_"; public static final String LUCENE_MATCH_VERSION_PARAM = "luceneMatchVersion"; public static final String MAX_CHARS = "maxChars"; public static final String NAME = "name"; diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java index 5a9c88b8a567..05021c4e7418 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java @@ -18,11 +18,10 @@ package org.apache.solr.update.processor; import java.io.IOException; -import java.util.Collection; import java.util.EnumSet; -import java.util.Locale; import java.util.Objects; +import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; import org.apache.solr.request.SolrQueryRequest; @@ -32,6 +31,7 @@ import static org.apache.solr.update.processor.DeeplyNestedUpdateProcessorFactory.NestedFlag; public class DeeplyNestedUpdateProcessor extends UpdateRequestProcessor { + private static final String splitChar = "\\."; private EnumSet fields; SolrQueryRequest req; @@ -53,7 +53,11 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { for(SolrInputField field: doc.values()) { for(Object val: field) { if(val instanceof SolrInputDocument) { - final String jointPath = Objects.isNull(fullPath) ? field.getName(): String.format(Locale.ROOT,"%s.%s", fullPath, field.getName()); + if(field.getName().contains(splitChar)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: " + field.getName() + + " contains: " + splitChar + " , which is reserved for the nested URP: "); + } + final String jointPath = Objects.isNull(fullPath) ? field.getName(): fullPath + splitChar + field.getName(); processChildDoc((SolrInputDocument) val, doc, jointPath); } } @@ -61,28 +65,15 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { } private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String fullPath) { - if(fields == NestedFlag.ALL) { + if(fields.contains(NestedFlag.PATH)) { setPathField(sdoc, fullPath); + } + if (fields.contains(NestedFlag.PARENT)) { setParentKey(sdoc, parent); - setLevelKey(sdoc, fullPath); - } else { - if(fields.contains(NestedFlag.PATH)) { - setPathField(sdoc, fullPath); - } - if (fields.contains(NestedFlag.PARENT)) { - setParentKey(sdoc, parent); - } - if(fields.contains(NestedFlag.LEVEL)) { - setLevelKey(sdoc, fullPath); - } } processDocChildren(sdoc, fullPath); } - private void setLevelKey(SolrInputDocument sdoc, String fullPath) { - sdoc.addField(IndexSchema.LEVEL_FIELD_NAME, fullPath.split("\\.").length); - } - private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { sdoc.addField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue(req.getSchema().getUniqueKeyField().getName())); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java index c63a01246da7..23338670cf10 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java @@ -17,6 +17,7 @@ package org.apache.solr.update.processor; +import java.util.Arrays; import java.util.EnumSet; import java.util.List; import java.util.Locale; @@ -34,7 +35,7 @@ public class DeeplyNestedUpdateProcessorFactory extends UpdateRequestProcessorFactory { private EnumSet fields; - private static final List allowedConfFields = NestedFlag.ALL.stream().map(e -> e.toString().toLowerCase(Locale.ROOT)).collect(Collectors.toList()); + private static final List allowedConfFields = Arrays.stream(NestedFlag.values()).map(e -> e.toString().toLowerCase(Locale.ROOT)).collect(Collectors.toList()); public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next ) { return new DeeplyNestedUpdateProcessor(req, rsp, fields, next); @@ -57,7 +58,7 @@ public void init( NamedList args ) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Deeply Nested URP may only contain: " + StringUtils.join(allowedConfFields, ", ") + " got: " + StringUtils.join(fields, ", ") + " instead"); } - this.fields = fields.size() == NestedFlag.values().length ? NestedFlag.ALL: generateNestedFlags(fields); + this.fields = generateNestedFlags(fields); } private static EnumSet generateNestedFlags(List fields) { @@ -66,9 +67,6 @@ private static EnumSet generateNestedFlags(List fields) { public enum NestedFlag { PATH, - PARENT, - LEVEL; - - public static final EnumSet ALL = EnumSet.allOf(NestedFlag.class); + PARENT } } diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml index b7c6b39da654..654acceac793 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml @@ -569,7 +569,6 @@ - diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml index bcce74c4d9af..4a4cf737bc39 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml @@ -32,7 +32,7 @@ - level,parent,path + parent,path diff --git a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java index 559b9f2e81b0..72bf381a153f 100644 --- a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java @@ -51,11 +51,6 @@ public void before() throws Exception { @Test public void testDeeplyNestedURPGrandChild() throws Exception { indexSampleData(); - assertJQ(req("q", IndexSchema.LEVEL_FIELD_NAME + ":2", - "fl","*", - "sort","id desc", - "wt","json"), - "/response/docs/[0]/id=='" + grandChildId + "'"); assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*.grandChild", "fl","*", @@ -68,11 +63,6 @@ public void testDeeplyNestedURPGrandChild() throws Exception { public void testDeeplyNestedURPChildren() throws Exception { final String[] childrenTests = {"/response/docs/[0]/id=='" + childrenIds[0] + "'", "/response/docs/[1]/id=='" + childrenIds[1] + "'"}; indexSampleData(); - assertJQ(req("q", IndexSchema.LEVEL_FIELD_NAME + ":1", - "fl","*", - "sort","id asc", - "wt","json"), - childrenTests); assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":children", "fl","*", From f2eac02c95d5fb674d6ae0918f30423c267b0fbe Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 11:06:05 +0300 Subject: [PATCH 09/34] SOLR-12441: raise exception if fieldName contains splitChar --- .../DeeplyNestedUpdateProcessor.java | 8 +- .../TestDeeplyNestedUpdateProcessor.java | 92 +++++++++++++------ 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java index 05021c4e7418..9636e805918c 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java @@ -31,7 +31,7 @@ import static org.apache.solr.update.processor.DeeplyNestedUpdateProcessorFactory.NestedFlag; public class DeeplyNestedUpdateProcessor extends UpdateRequestProcessor { - private static final String splitChar = "\\."; + public static final String splitChar = "."; private EnumSet fields; SolrQueryRequest req; @@ -54,10 +54,10 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { for(Object val: field) { if(val instanceof SolrInputDocument) { if(field.getName().contains(splitChar)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: " + field.getName() - + " contains: " + splitChar + " , which is reserved for the nested URP: "); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: '" + field.getName() + + "' contains: '" + splitChar + "' , which is reserved for the nested URP"); } - final String jointPath = Objects.isNull(fullPath) ? field.getName(): fullPath + splitChar + field.getName(); + final String jointPath = Objects.isNull(fullPath) ? field.getName(): String.join(splitChar, fullPath, field.getName()); processChildDoc((SolrInputDocument) val, doc, jointPath); } } diff --git a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java index 72bf381a153f..a533fe8d7642 100644 --- a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.request.SolrQueryRequest; @@ -30,12 +31,64 @@ import org.apache.solr.servlet.SolrRequestParsers; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.apache.solr.update.processor.DeeplyNestedUpdateProcessor.splitChar; public class TestDeeplyNestedUpdateProcessor extends SolrTestCaseJ4 { public static final String[] childrenIds = { "2", "3" }; public static final String grandChildId = "4"; + private static final String jDoc = "{\n" + + " \"add\": {\n" + + " \"doc\": {\n" + + " \"id\": \"1\",\n" + + " \"children\": [\n" + + " {\n" + + " \"id\": \"2\",\n" + + " \"foo_s\": \"Yaz\"\n" + + " \"grandChild\": \n" + + " {\n" + + " \"id\": \""+ grandChildId + "\",\n" + + " \"foo_s\": \"Jazz\"\n" + + " },\n" + + " },\n" + + " {\n" + + " \"id\": \"3\",\n" + + " \"foo_s\": \"Bar\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; + + private static final String errDoc = "{\n" + + " \"add\": {\n" + + " \"doc\": {\n" + + " \"id\": \"1\",\n" + + " \"children" + splitChar + "a\": [\n" + + " {\n" + + " \"id\": \"2\",\n" + + " \"foo_s\": \"Yaz\"\n" + + " \"grandChild\": \n" + + " {\n" + + " \"id\": \""+ grandChildId + "\",\n" + + " \"foo_s\": \"Jazz\"\n" + + " },\n" + + " },\n" + + " {\n" + + " \"id\": \"3\",\n" + + " \"foo_s\": \"Bar\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; + + @Rule + public ExpectedException thrown = ExpectedException.none(); @BeforeClass public static void beforeClass() throws Exception { @@ -50,7 +103,7 @@ public void before() throws Exception { @Test public void testDeeplyNestedURPGrandChild() throws Exception { - indexSampleData(); + indexSampleData(jDoc); assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*.grandChild", "fl","*", @@ -62,7 +115,7 @@ public void testDeeplyNestedURPGrandChild() throws Exception { @Test public void testDeeplyNestedURPChildren() throws Exception { final String[] childrenTests = {"/response/docs/[0]/id=='" + childrenIds[0] + "'", "/response/docs/[1]/id=='" + childrenIds[1] + "'"}; - indexSampleData(); + indexSampleData(jDoc); assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":children", "fl","*", @@ -71,32 +124,19 @@ public void testDeeplyNestedURPChildren() throws Exception { childrenTests); } - private void indexSampleData() throws Exception { - final String jDoc = "{\n" + - " \"add\": {\n" + - " \"doc\": {\n" + - " \"id\": \"1\",\n" + - " \"children\": [\n" + - " {\n" + - " \"id\": \"2\",\n" + - " \"foo_s\": \"Yaz\"\n" + - " \"grandChild\": \n" + - " {\n" + - " \"id\": \""+ grandChildId + "\",\n" + - " \"foo_s\": \"Jazz\"\n" + - " },\n" + - " },\n" + - " {\n" + - " \"id\": \"3\",\n" + - " \"foo_s\": \"Bar\"\n" + - " }\n" + - " ]\n" + - " }\n" + - " }\n" + - "}"; + @Test + public void testDeeplyNestedURPFieldNameException() throws Exception { + final String errMsg = "Field name: children.a contains: '.' , which is reserved for the nested URP"; + thrown.expect(SolrException.class); + indexSampleData(errDoc); + thrown.expectMessage(errMsg); + } + + private void indexSampleData(String cmd) throws Exception { + List streams = new ArrayList<>( 1 ); - streams.add( new ContentStreamBase.StringStream( jDoc ) ); + streams.add( new ContentStreamBase.StringStream( cmd ) ); SolrQueryRequest req = null; try { From 60948a9b8ed2977051157c3c4fea8cfb722caf7f Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 11:25:56 +0300 Subject: [PATCH 10/34] SOLR-12441: change addField to setField --- .../solr/update/processor/DeeplyNestedUpdateProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java index 9636e805918c..b9df0228cc18 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java @@ -75,11 +75,11 @@ private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, S } private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { - sdoc.addField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue(req.getSchema().getUniqueKeyField().getName())); + sdoc.setField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue(req.getSchema().getUniqueKeyField().getName())); } private void setPathField(SolrInputDocument sdoc, String fullPath) { - sdoc.addField(IndexSchema.PATH_FIELD_NAME, fullPath); + sdoc.setField(IndexSchema.PATH_FIELD_NAME, fullPath); } } \ No newline at end of file From 37e61b9be4b8d876a0964284f4c7067695ed3b4a Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 11:37:58 +0300 Subject: [PATCH 11/34] SOLR-12441: make nestedurp test exception less doc specific --- .../org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java index a533fe8d7642..deb286c65d22 100644 --- a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java @@ -126,7 +126,7 @@ public void testDeeplyNestedURPChildren() throws Exception { @Test public void testDeeplyNestedURPFieldNameException() throws Exception { - final String errMsg = "Field name: children.a contains: '.' , which is reserved for the nested URP"; + final String errMsg = "contains: '" + splitChar + "' , which is reserved for the nested URP"; thrown.expect(SolrException.class); indexSampleData(errDoc); thrown.expectMessage(errMsg); From 8113edce7a08de7b4c1deb7972ac1f5e822367d7 Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 12:39:44 +0300 Subject: [PATCH 12/34] SOLR-12441: rename to nestedUpdateProcessor --- ...tedUpdateProcessor.java => NestedUpdateProcessor.java} | 6 +++--- ...ssorFactory.java => NestedUpdateProcessorFactory.java} | 4 ++-- .../src/test-files/solr/collection1/conf/schema15.xml | 2 +- ...pdateProcessor.java => TestNestedUpdateProcessor.java} | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) rename solr/core/src/java/org/apache/solr/update/processor/{DeeplyNestedUpdateProcessor.java => NestedUpdateProcessor.java} (90%) rename solr/core/src/java/org/apache/solr/update/processor/{DeeplyNestedUpdateProcessorFactory.java => NestedUpdateProcessorFactory.java} (94%) rename solr/core/src/test/org/apache/solr/update/{TestDeeplyNestedUpdateProcessor.java => TestNestedUpdateProcessor.java} (95%) diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java similarity index 90% rename from solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java rename to solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java index b9df0228cc18..920ad4be3154 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java @@ -28,15 +28,15 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.IndexSchema; import org.apache.solr.update.AddUpdateCommand; -import static org.apache.solr.update.processor.DeeplyNestedUpdateProcessorFactory.NestedFlag; +import static org.apache.solr.update.processor.NestedUpdateProcessorFactory.NestedFlag; -public class DeeplyNestedUpdateProcessor extends UpdateRequestProcessor { +public class NestedUpdateProcessor extends UpdateRequestProcessor { public static final String splitChar = "."; private EnumSet fields; SolrQueryRequest req; - protected DeeplyNestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, EnumSet fields, UpdateRequestProcessor next) { + protected NestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, EnumSet fields, UpdateRequestProcessor next) { super(next); this.req = req; this.fields = fields; diff --git a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java similarity index 94% rename from solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java rename to solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 23338670cf10..dc23a801674c 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DeeplyNestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -32,13 +32,13 @@ import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; -public class DeeplyNestedUpdateProcessorFactory extends UpdateRequestProcessorFactory { +public class NestedUpdateProcessorFactory extends UpdateRequestProcessorFactory { private EnumSet fields; private static final List allowedConfFields = Arrays.stream(NestedFlag.values()).map(e -> e.toString().toLowerCase(Locale.ROOT)).collect(Collectors.toList()); public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next ) { - return new DeeplyNestedUpdateProcessor(req, rsp, fields, next); + return new NestedUpdateProcessor(req, rsp, fields, next); } @Override diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml index 654acceac793..37a49d6d122e 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml @@ -568,7 +568,7 @@ - + diff --git a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java similarity index 95% rename from solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java rename to solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index deb286c65d22..bbd57d5823c5 100644 --- a/solr/core/src/test/org/apache/solr/update/TestDeeplyNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -35,12 +35,12 @@ import org.junit.Test; import org.junit.rules.ExpectedException; -import static org.apache.solr.update.processor.DeeplyNestedUpdateProcessor.splitChar; +import static org.apache.solr.update.processor.NestedUpdateProcessor.splitChar; -public class TestDeeplyNestedUpdateProcessor extends SolrTestCaseJ4 { +public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { - public static final String[] childrenIds = { "2", "3" }; - public static final String grandChildId = "4"; + private static final String[] childrenIds = { "2", "3" }; + private static final String grandChildId = "4"; private static final String jDoc = "{\n" + " \"add\": {\n" + " \"doc\": {\n" + From f57d30eb6b1cc21f5fce0086a714834027c08173 Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 13:11:20 +0300 Subject: [PATCH 13/34] SOLR-12441: config param fix error message --- .../solr/update/processor/NestedUpdateProcessorFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index dc23a801674c..85e08f785fa6 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -47,11 +47,11 @@ public void init( NamedList args ) Object tmp = args.remove("fields"); if (null == tmp) { throw new SolrException(SERVER_ERROR, - "'versionField' must be configured"); + "'fields' must be configured"); } if (! (tmp instanceof String) ) { throw new SolrException(SERVER_ERROR, - "'versionField' must be configured as a "); + "'fields' must be configured as a "); } List fields = StrUtils.splitSmart((String)tmp, ','); if(!allowedConfFields.containsAll(fields)) { From f9c6393efe30a75d1d6fff8042eb31afaaa17b58 Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 16:49:12 +0300 Subject: [PATCH 14/34] SOLR-12441: remove "Deeply" from urp --- .../solr/update/processor/NestedUpdateProcessorFactory.java | 2 +- .../collection1/conf/solrconfig-update-processor-chains.xml | 4 ++-- .../org/apache/solr/update/TestNestedUpdateProcessor.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 85e08f785fa6..467dd3b756a5 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -55,7 +55,7 @@ public void init( NamedList args ) } List fields = StrUtils.splitSmart((String)tmp, ','); if(!allowedConfFields.containsAll(fields)) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Deeply Nested URP may only contain: " + StringUtils.join(allowedConfFields, ", ") + + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Nested URP may only contain: " + StringUtils.join(allowedConfFields, ", ") + " got: " + StringUtils.join(fields, ", ") + " instead"); } this.fields = generateNestedFlags(fields); diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml index 4a4cf737bc39..6502c00295f5 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml @@ -30,8 +30,8 @@ - - + + parent,path diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index bbd57d5823c5..b8c2965a44cc 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -140,7 +140,7 @@ private void indexSampleData(String cmd) throws Exception { SolrQueryRequest req = null; try { - req = new SolrRequestParsers(h.getCore().getSolrConfig()).buildRequestFrom( h.getCore(), params("update.chain", "deeply-nested"), streams ); + req = new SolrRequestParsers(h.getCore().getSolrConfig()).buildRequestFrom( h.getCore(), params("update.chain", "nested"), streams ); SolrQueryResponse rsp = new SolrQueryResponse(); SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); h.getCore().execute( h.getCore().getRequestHandler("/update"), req, rsp ); From c8d83da41d92862395ce8b0c68c829c15096f477 Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 17:42:32 +0300 Subject: [PATCH 15/34] SOLR-12441: replace EnumSet config with two booleans --- .../processor/NestedUpdateProcessor.java | 22 ++++---- .../NestedUpdateProcessorFactory.java | 50 ++++--------------- .../solrconfig-update-processor-chains.xml | 4 +- .../update/TestNestedUpdateProcessor.java | 9 ++-- 4 files changed, 26 insertions(+), 59 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java index 920ad4be3154..68d16c251c37 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java @@ -18,7 +18,6 @@ package org.apache.solr.update.processor; import java.io.IOException; -import java.util.EnumSet; import java.util.Objects; import org.apache.solr.common.SolrException; @@ -28,18 +27,19 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.IndexSchema; import org.apache.solr.update.AddUpdateCommand; -import static org.apache.solr.update.processor.NestedUpdateProcessorFactory.NestedFlag; public class NestedUpdateProcessor extends UpdateRequestProcessor { - public static final String splitChar = "."; - private EnumSet fields; + public static final String PATH_SEP_CHAR = "/"; + private boolean storePath; + private boolean storeParent; SolrQueryRequest req; - protected NestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, EnumSet fields, UpdateRequestProcessor next) { + protected NestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, boolean storeParent, boolean storePath, UpdateRequestProcessor next) { super(next); this.req = req; - this.fields = fields; + this.storeParent = storeParent; + this.storePath = storePath; } @Override @@ -53,11 +53,11 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { for(SolrInputField field: doc.values()) { for(Object val: field) { if(val instanceof SolrInputDocument) { - if(field.getName().contains(splitChar)) { + if(field.getName().contains(PATH_SEP_CHAR)) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: '" + field.getName() - + "' contains: '" + splitChar + "' , which is reserved for the nested URP"); + + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); } - final String jointPath = Objects.isNull(fullPath) ? field.getName(): String.join(splitChar, fullPath, field.getName()); + final String jointPath = Objects.isNull(fullPath) ? field.getName(): String.join(PATH_SEP_CHAR, fullPath, field.getName()); processChildDoc((SolrInputDocument) val, doc, jointPath); } } @@ -65,10 +65,10 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { } private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String fullPath) { - if(fields.contains(NestedFlag.PATH)) { + if(storePath) { setPathField(sdoc, fullPath); } - if (fields.contains(NestedFlag.PARENT)) { + if (storeParent) { setParentKey(sdoc, parent); } processDocChildren(sdoc, fullPath); diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 467dd3b756a5..defd778c8db2 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -17,56 +17,26 @@ package org.apache.solr.update.processor; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.List; -import java.util.Locale; -import java.util.stream.Collectors; - -import org.apache.solr.common.SolrException; -import org.apache.commons.lang3.StringUtils; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.StrUtils; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; - -import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; +import org.apache.solr.schema.IndexSchema; public class NestedUpdateProcessorFactory extends UpdateRequestProcessorFactory { - private EnumSet fields; - private static final List allowedConfFields = Arrays.stream(NestedFlag.values()).map(e -> e.toString().toLowerCase(Locale.ROOT)).collect(Collectors.toList()); - public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next ) { - return new NestedUpdateProcessor(req, rsp, fields, next); - } - - @Override - public void init( NamedList args ) - { - Object tmp = args.remove("fields"); - if (null == tmp) { - throw new SolrException(SERVER_ERROR, - "'fields' must be configured"); - } - if (! (tmp instanceof String) ) { - throw new SolrException(SERVER_ERROR, - "'fields' must be configured as a "); - } - List fields = StrUtils.splitSmart((String)tmp, ','); - if(!allowedConfFields.containsAll(fields)) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Nested URP may only contain: " + StringUtils.join(allowedConfFields, ", ") + - " got: " + StringUtils.join(fields, ", ") + " instead"); + boolean storeParent = shouldStoreDocParent(req.getSchema()); + boolean storePath = shouldStoreDocPath(req.getSchema()); + if(!(storeParent || storePath)) { + return next; } - this.fields = generateNestedFlags(fields); + return new NestedUpdateProcessor(req, rsp, shouldStoreDocParent(req.getSchema()), shouldStoreDocPath(req.getSchema()), next); } - private static EnumSet generateNestedFlags(List fields) { - return EnumSet.copyOf(fields.stream().map(e -> NestedFlag.valueOf(e.toUpperCase(Locale.ROOT))).collect(Collectors.toList())); + private static boolean shouldStoreDocParent(IndexSchema schema) { + return schema.getFields().containsKey(IndexSchema.PARENT_FIELD_NAME); } - public enum NestedFlag { - PATH, - PARENT + private static boolean shouldStoreDocPath(IndexSchema schema) { + return schema.getFields().containsKey(IndexSchema.PATH_FIELD_NAME); } } diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml index 6502c00295f5..f22354e4ced2 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml @@ -31,9 +31,7 @@ - - parent,path - + diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index b8c2965a44cc..33b0dfd70e3e 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -35,10 +35,9 @@ import org.junit.Test; import org.junit.rules.ExpectedException; -import static org.apache.solr.update.processor.NestedUpdateProcessor.splitChar; - public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { + private static final char PATH_SEP_CHAR = '/'; private static final String[] childrenIds = { "2", "3" }; private static final String grandChildId = "4"; private static final String jDoc = "{\n" + @@ -68,7 +67,7 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { " \"add\": {\n" + " \"doc\": {\n" + " \"id\": \"1\",\n" + - " \"children" + splitChar + "a\": [\n" + + " \"children" + PATH_SEP_CHAR + "a\": [\n" + " {\n" + " \"id\": \"2\",\n" + " \"foo_s\": \"Yaz\"\n" + @@ -105,7 +104,7 @@ public void before() throws Exception { public void testDeeplyNestedURPGrandChild() throws Exception { indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*.grandChild", + assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*/grandChild", "fl","*", "sort","id desc", "wt","json"), @@ -126,7 +125,7 @@ public void testDeeplyNestedURPChildren() throws Exception { @Test public void testDeeplyNestedURPFieldNameException() throws Exception { - final String errMsg = "contains: '" + splitChar + "' , which is reserved for the nested URP"; + final String errMsg = "contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"; thrown.expect(SolrException.class); indexSampleData(errDoc); thrown.expectMessage(errMsg); From 4a7e40ea4211c3c11370a3535bc22521d47c79a2 Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 26 Jun 2018 17:46:29 +0300 Subject: [PATCH 16/34] SOLR-12441: all test use PATH_SEP_CHAR constant --- .../test/org/apache/solr/update/TestNestedUpdateProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index 33b0dfd70e3e..8ad0b235110f 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -104,7 +104,7 @@ public void before() throws Exception { public void testDeeplyNestedURPGrandChild() throws Exception { indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*/grandChild", + assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*" + PATH_SEP_CHAR + "grandChild", "fl","*", "sort","id desc", "wt","json"), From ce6fc2eadefbfcfe968b455bd8218387180e692b Mon Sep 17 00:00:00 2001 From: Moshe Date: Thu, 28 Jun 2018 08:36:37 +0300 Subject: [PATCH 17/34] SOLR-12441: failing test for idless child docs --- .../processor/NestedUpdateProcessor.java | 21 +++++++++--- .../update/TestNestedUpdateProcessor.java | 32 +++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java index 68d16c251c37..7e6f95a26774 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java @@ -45,11 +45,13 @@ protected NestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, boo @Override public void processAdd(AddUpdateCommand cmd) throws IOException { SolrInputDocument doc = cmd.getSolrInputDocument(); - processDocChildren(doc, null); + String rootId = doc.getField(req.getSchema().getUniqueKeyField().getName()).getFirstValue().toString(); + processDocChildren(doc, rootId, null); super.processAdd(cmd); } - private void processDocChildren(SolrInputDocument doc, String fullPath) { + private void processDocChildren(SolrInputDocument doc, String rootId, String fullPath) { + int childNum = 0; for(SolrInputField field: doc.values()) { for(Object val: field) { if(val instanceof SolrInputDocument) { @@ -58,20 +60,29 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); } final String jointPath = Objects.isNull(fullPath) ? field.getName(): String.join(PATH_SEP_CHAR, fullPath, field.getName()); - processChildDoc((SolrInputDocument) val, doc, jointPath); + SolrInputDocument cDoc = (SolrInputDocument) val; + if(!cDoc.containsKey(req.getSchema().getUniqueKeyField().getName())) { + cDoc.setField(req.getSchema().getUniqueKeyField().getName(), generateChildUniqueId(rootId, jointPath, childNum)); + } + processChildDoc((SolrInputDocument) val, doc, rootId, jointPath); } + ++childNum; } } } - private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String fullPath) { + private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String rootId, String fullPath) { if(storePath) { setPathField(sdoc, fullPath); } if (storeParent) { setParentKey(sdoc, parent); } - processDocChildren(sdoc, fullPath); + processDocChildren(sdoc, rootId, fullPath); + } + + private String generateChildUniqueId(String rootId, String childPath, int childNum) { + return String.join(PATH_SEP_CHAR, rootId, childPath, Integer.toString(childNum)); } private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index 8ad0b235110f..0cba5ecca031 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -63,6 +63,26 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { " }\n" + "}"; + private static final String noIdChildren = "{\n" + + " \"add\": {\n" + + " \"doc\": {\n" + + " \"id\": \"1\",\n" + + " \"children\": [\n" + + " {\n" + + " \"foo_s\": \"Yaz\"\n" + + " \"grandChild\": \n" + + " {\n" + + " \"foo_s\": \"Jazz\"\n" + + " },\n" + + " },\n" + + " {\n" + + " \"foo_s\": \"Bar\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " }\n" + + "}"; + private static final String errDoc = "{\n" + " \"add\": {\n" + " \"doc\": {\n" + @@ -123,6 +143,18 @@ public void testDeeplyNestedURPChildren() throws Exception { childrenTests); } + @Test + public void testDeeplyNestedURPChildrenWoId() throws Exception { + final String[] childrenTests = {"/response/docs/[0]/id=='1" + PATH_SEP_CHAR + "children" + PATH_SEP_CHAR + "2'"}; + indexSampleData(noIdChildren); + + assertJQ(req("q", "foo_s:Bar", + "fl","*", + "sort","id asc", + "wt","json"), + childrenTests); + } + @Test public void testDeeplyNestedURPFieldNameException() throws Exception { final String errMsg = "contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"; From aaed861dc6f672642a9f9ccc65ab727088c1d265 Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 2 Jul 2018 09:26:26 +0300 Subject: [PATCH 18/34] SOLR-12441: req.getSchema().getUniqueKeyField().getName() as field --- .../update/processor/NestedUpdateProcessor.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java index 7e6f95a26774..12932663a362 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java @@ -29,23 +29,23 @@ import org.apache.solr.update.AddUpdateCommand; public class NestedUpdateProcessor extends UpdateRequestProcessor { - public static final String PATH_SEP_CHAR = "/"; + private static final String PATH_SEP_CHAR = "/"; private boolean storePath; private boolean storeParent; - SolrQueryRequest req; + private String uniqueKeyFieldName; protected NestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, boolean storeParent, boolean storePath, UpdateRequestProcessor next) { super(next); - this.req = req; this.storeParent = storeParent; this.storePath = storePath; + this.uniqueKeyFieldName = req.getSchema().getUniqueKeyField().getName(); } @Override public void processAdd(AddUpdateCommand cmd) throws IOException { SolrInputDocument doc = cmd.getSolrInputDocument(); - String rootId = doc.getField(req.getSchema().getUniqueKeyField().getName()).getFirstValue().toString(); + String rootId = doc.getField(uniqueKeyFieldName).getFirstValue().toString(); processDocChildren(doc, rootId, null); super.processAdd(cmd); } @@ -61,8 +61,8 @@ private void processDocChildren(SolrInputDocument doc, String rootId, String ful } final String jointPath = Objects.isNull(fullPath) ? field.getName(): String.join(PATH_SEP_CHAR, fullPath, field.getName()); SolrInputDocument cDoc = (SolrInputDocument) val; - if(!cDoc.containsKey(req.getSchema().getUniqueKeyField().getName())) { - cDoc.setField(req.getSchema().getUniqueKeyField().getName(), generateChildUniqueId(rootId, jointPath, childNum)); + if(!cDoc.containsKey(uniqueKeyFieldName)) { + cDoc.setField(uniqueKeyFieldName, generateChildUniqueId(rootId, jointPath, childNum)); } processChildDoc((SolrInputDocument) val, doc, rootId, jointPath); } @@ -86,7 +86,7 @@ private String generateChildUniqueId(String rootId, String childPath, int childN } private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { - sdoc.setField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue(req.getSchema().getUniqueKeyField().getName())); + sdoc.setField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue(uniqueKeyFieldName)); } private void setPathField(SolrInputDocument sdoc, String fullPath) { From bcf5261dd0c69d7bf3774565dbb3421a55bc3e3b Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 2 Jul 2018 09:49:03 +0300 Subject: [PATCH 19/34] SOLR-12441: combined moved NestedUpdateProcessor into NestedUpdateProcessorFactory --- .../processor/NestedUpdateProcessor.java | 96 ------------------- .../NestedUpdateProcessorFactory.java | 76 +++++++++++++++ 2 files changed, 76 insertions(+), 96 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java deleted file mode 100644 index 12932663a362..000000000000 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessor.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.update.processor; - -import java.io.IOException; -import java.util.Objects; - -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.SolrInputField; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.update.AddUpdateCommand; - -public class NestedUpdateProcessor extends UpdateRequestProcessor { - private static final String PATH_SEP_CHAR = "/"; - private boolean storePath; - private boolean storeParent; - private String uniqueKeyFieldName; - - - protected NestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, boolean storeParent, boolean storePath, UpdateRequestProcessor next) { - super(next); - this.storeParent = storeParent; - this.storePath = storePath; - this.uniqueKeyFieldName = req.getSchema().getUniqueKeyField().getName(); - } - - @Override - public void processAdd(AddUpdateCommand cmd) throws IOException { - SolrInputDocument doc = cmd.getSolrInputDocument(); - String rootId = doc.getField(uniqueKeyFieldName).getFirstValue().toString(); - processDocChildren(doc, rootId, null); - super.processAdd(cmd); - } - - private void processDocChildren(SolrInputDocument doc, String rootId, String fullPath) { - int childNum = 0; - for(SolrInputField field: doc.values()) { - for(Object val: field) { - if(val instanceof SolrInputDocument) { - if(field.getName().contains(PATH_SEP_CHAR)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: '" + field.getName() - + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); - } - final String jointPath = Objects.isNull(fullPath) ? field.getName(): String.join(PATH_SEP_CHAR, fullPath, field.getName()); - SolrInputDocument cDoc = (SolrInputDocument) val; - if(!cDoc.containsKey(uniqueKeyFieldName)) { - cDoc.setField(uniqueKeyFieldName, generateChildUniqueId(rootId, jointPath, childNum)); - } - processChildDoc((SolrInputDocument) val, doc, rootId, jointPath); - } - ++childNum; - } - } - } - - private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String rootId, String fullPath) { - if(storePath) { - setPathField(sdoc, fullPath); - } - if (storeParent) { - setParentKey(sdoc, parent); - } - processDocChildren(sdoc, rootId, fullPath); - } - - private String generateChildUniqueId(String rootId, String childPath, int childNum) { - return String.join(PATH_SEP_CHAR, rootId, childPath, Integer.toString(childNum)); - } - - private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { - sdoc.setField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue(uniqueKeyFieldName)); - } - - private void setPathField(SolrInputDocument sdoc, String fullPath) { - sdoc.setField(IndexSchema.PATH_FIELD_NAME, fullPath); - } - -} \ No newline at end of file diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index defd778c8db2..375a2f1e70c7 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -17,9 +17,15 @@ package org.apache.solr.update.processor; +import java.io.IOException; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.SolrInputField; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.update.AddUpdateCommand; public class NestedUpdateProcessorFactory extends UpdateRequestProcessorFactory { @@ -40,3 +46,73 @@ private static boolean shouldStoreDocPath(IndexSchema schema) { return schema.getFields().containsKey(IndexSchema.PATH_FIELD_NAME); } } + +class NestedUpdateProcessor extends UpdateRequestProcessor { + private static final String PATH_SEP_CHAR = "/"; + private boolean storePath; + private boolean storeParent; + private String uniqueKeyFieldName; + + + protected NestedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, boolean storeParent, boolean storePath, UpdateRequestProcessor next) { + super(next); + this.storeParent = storeParent; + this.storePath = storePath; + this.uniqueKeyFieldName = req.getSchema().getUniqueKeyField().getName(); + } + + @Override + public void processAdd(AddUpdateCommand cmd) throws IOException { + SolrInputDocument doc = cmd.getSolrInputDocument(); + processDocChildren(doc, null); + super.processAdd(cmd); + } + + private void processDocChildren(SolrInputDocument doc, String fullPath) { + int childNum = 0; + for(SolrInputField field: doc.values()) { + for(Object val: field) { + if(!(val instanceof SolrInputDocument)) { + // either all collection items are child docs or none are. + break; + } + + if(field.getName().contains(PATH_SEP_CHAR)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: '" + field.getName() + + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); + } + final String jointPath = fullPath == null ? field.getName(): String.join(PATH_SEP_CHAR, fullPath, field.getName()); + SolrInputDocument cDoc = (SolrInputDocument) val; + if(!cDoc.containsKey(uniqueKeyFieldName)) { + String parentDocId = doc.getField(uniqueKeyFieldName).getFirstValue().toString(); + cDoc.setField(uniqueKeyFieldName, generateChildUniqueId(parentDocId, jointPath, childNum)); + } + processChildDoc((SolrInputDocument) val, doc, jointPath); + ++childNum; + } + } + } + + private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, String fullPath) { + if(storePath) { + setPathField(sdoc, fullPath); + } + if (storeParent) { + setParentKey(sdoc, parent); + } + processDocChildren(sdoc, fullPath); + } + + private String generateChildUniqueId(String parentId, String childPath, int childNum) { + return String.join(PATH_SEP_CHAR, parentId, childPath, Integer.toString(childNum)); + } + + private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { + sdoc.setField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue(uniqueKeyFieldName)); + } + + private void setPathField(SolrInputDocument sdoc, String fullPath) { + sdoc.setField(IndexSchema.PATH_FIELD_NAME, fullPath); + } + +} From fe4d1a0b5d396b3c790405ba5d48bd7adbc93124 Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 2 Jul 2018 10:10:36 +0300 Subject: [PATCH 20/34] SOLR-12441: use updateJ for NestedUpdateProcessor tests --- .../NestedUpdateProcessorFactory.java | 1 + .../update/TestNestedUpdateProcessor.java | 30 +------------------ 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 375a2f1e70c7..1d4aa3454ca8 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -104,6 +104,7 @@ private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, S } private String generateChildUniqueId(String parentId, String childPath, int childNum) { + // combines parentId with the path and childNum. e.g. "3/book/pages/footnote/1" return String.join(PATH_SEP_CHAR, parentId, childPath, Integer.toString(childNum)); } diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index 0cba5ecca031..61dab5efe55c 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -17,18 +17,9 @@ package org.apache.solr.update; -import java.util.ArrayList; -import java.util.List; - import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.ContentStream; -import org.apache.solr.common.util.ContentStreamBase; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrRequestInfo; -import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.schema.IndexSchema; -import org.apache.solr.servlet.SolrRequestParsers; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; @@ -164,26 +155,7 @@ public void testDeeplyNestedURPFieldNameException() throws Exception { } private void indexSampleData(String cmd) throws Exception { - - - List streams = new ArrayList<>( 1 ); - streams.add( new ContentStreamBase.StringStream( cmd ) ); - - SolrQueryRequest req = null; - try { - req = new SolrRequestParsers(h.getCore().getSolrConfig()).buildRequestFrom( h.getCore(), params("update.chain", "nested"), streams ); - SolrQueryResponse rsp = new SolrQueryResponse(); - SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); - h.getCore().execute( h.getCore().getRequestHandler("/update"), req, rsp ); - if( rsp.getException() != null ) { - throw rsp.getException(); - } - } finally { - if (req != null) { - req.close(); - SolrRequestInfo.clearRequestInfo(); - } - } + updateJ(cmd, params("update.chain", "nested")); assertU(commit()); } } From cf10ecbcb320882fdd186c07b077409fabce0ad9 Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 3 Jul 2018 14:29:50 +0300 Subject: [PATCH 21/34] SOLR-12441: use string concatenation --- .../solr/update/processor/NestedUpdateProcessorFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 1d4aa3454ca8..52c8158950ac 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -81,7 +81,7 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: '" + field.getName() + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); } - final String jointPath = fullPath == null ? field.getName(): String.join(PATH_SEP_CHAR, fullPath, field.getName()); + final String jointPath = fullPath == null ? field.getName(): fullPath + PATH_SEP_CHAR + field.getName(); SolrInputDocument cDoc = (SolrInputDocument) val; if(!cDoc.containsKey(uniqueKeyFieldName)) { String parentDocId = doc.getField(uniqueKeyFieldName).getFirstValue().toString(); From b764bd6d7df2d194425e48653a5f62575ed3ba9a Mon Sep 17 00:00:00 2001 From: Moshe Date: Wed, 4 Jul 2018 16:56:28 +0300 Subject: [PATCH 22/34] SOLR-12441: fix nestedupdateprocessor id generator algorithm --- .../processor/NestedUpdateProcessorFactory.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 52c8158950ac..8e59888845f5 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -76,16 +76,17 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { // either all collection items are child docs or none are. break; } + final String fieldName = field.getName(); - if(field.getName().contains(PATH_SEP_CHAR)) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: '" + field.getName() + if(fieldName.contains(PATH_SEP_CHAR)) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: '" + fieldName + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); } - final String jointPath = fullPath == null ? field.getName(): fullPath + PATH_SEP_CHAR + field.getName(); + final String jointPath = fullPath == null ? fieldName: fullPath + PATH_SEP_CHAR + fieldName; SolrInputDocument cDoc = (SolrInputDocument) val; if(!cDoc.containsKey(uniqueKeyFieldName)) { String parentDocId = doc.getField(uniqueKeyFieldName).getFirstValue().toString(); - cDoc.setField(uniqueKeyFieldName, generateChildUniqueId(parentDocId, jointPath, childNum)); + cDoc.setField(uniqueKeyFieldName, generateChildUniqueId(parentDocId, fieldName, childNum)); } processChildDoc((SolrInputDocument) val, doc, jointPath); ++childNum; @@ -103,9 +104,9 @@ private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, S processDocChildren(sdoc, fullPath); } - private String generateChildUniqueId(String parentId, String childPath, int childNum) { - // combines parentId with the path and childNum. e.g. "3/book/pages/footnote/1" - return String.join(PATH_SEP_CHAR, parentId, childPath, Integer.toString(childNum)); + private String generateChildUniqueId(String parentId, String childKey, int childNum) { + // combines parentId with the child's key and childNum. e.g. "10/footnote/1" + return String.join(PATH_SEP_CHAR, parentId, childKey, Integer.toString(childNum)); } private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { From 8f138f97b3243b8c1a8b85ae156931f0c98a0b3e Mon Sep 17 00:00:00 2001 From: Moshe Date: Wed, 4 Jul 2018 16:58:29 +0300 Subject: [PATCH 23/34] SOLR-12441: fix id less child docs test for nestedUpdateProcessor --- .../update/TestNestedUpdateProcessor.java | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index 61dab5efe55c..35124739b0e9 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -17,9 +17,14 @@ package org.apache.solr.update; +import java.util.List; + import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.update.processor.NestedUpdateProcessorFactory; +import org.apache.solr.update.processor.UpdateRequestProcessor; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; @@ -136,14 +141,19 @@ public void testDeeplyNestedURPChildren() throws Exception { @Test public void testDeeplyNestedURPChildrenWoId() throws Exception { - final String[] childrenTests = {"/response/docs/[0]/id=='1" + PATH_SEP_CHAR + "children" + PATH_SEP_CHAR + "2'"}; - indexSampleData(noIdChildren); - - assertJQ(req("q", "foo_s:Bar", - "fl","*", - "sort","id asc", - "wt","json"), - childrenTests); + final String parentId = "3"; + final String childKey = "grandChild"; + final String expectedId = parentId + PATH_SEP_CHAR + childKey + PATH_SEP_CHAR + "0"; + SolrInputDocument noIdChildren = sdoc("id", "1", "children", sdocs(sdoc("id", "2", "name_s", "Yaz"), sdoc("id", parentId, "name_s", "Jazz", childKey, sdoc("names_s", "Gaz")))); + UpdateRequestProcessor nestedUpdate = new NestedUpdateProcessorFactory().getInstance(req(), null, null); + AddUpdateCommand cmd = new AddUpdateCommand(req()); + cmd.solrDoc = noIdChildren; + nestedUpdate.processAdd(cmd); + cmd.clear(); + List children = (List) noIdChildren.get("children").getValues(); + SolrInputDocument idLessChild = (SolrInputDocument)((SolrInputDocument) children.get(1)).get(childKey).getValue(); + assertTrue("Id less child did not get an Id", idLessChild.containsKey("id")); + assertEquals("Id less child was assigned an unexpected id", expectedId, idLessChild.getFieldValue("id").toString()); } @Test From cf3ef23f85e3238d9b8080b0a13159163e4033a0 Mon Sep 17 00:00:00 2001 From: Moshe Date: Thu, 5 Jul 2018 08:02:33 +0300 Subject: [PATCH 24/34] SOLR-12441: use string concat for NestedUpdateProcessor.generateChildUniqueId --- .../update/processor/NestedUpdateProcessorFactory.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 8e59888845f5..a14959d1a70d 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -82,11 +82,13 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: '" + fieldName + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); } - final String jointPath = fullPath == null ? fieldName: fullPath + PATH_SEP_CHAR + fieldName; + final String sChildNum = String.valueOf(childNum); + final String lastPath = fieldName + PATH_SEP_CHAR + sChildNum + PATH_SEP_CHAR; + final String jointPath = fullPath == null ? lastPath : fullPath + PATH_SEP_CHAR + lastPath; SolrInputDocument cDoc = (SolrInputDocument) val; if(!cDoc.containsKey(uniqueKeyFieldName)) { String parentDocId = doc.getField(uniqueKeyFieldName).getFirstValue().toString(); - cDoc.setField(uniqueKeyFieldName, generateChildUniqueId(parentDocId, fieldName, childNum)); + cDoc.setField(uniqueKeyFieldName, generateChildUniqueId(parentDocId, fieldName, sChildNum)); } processChildDoc((SolrInputDocument) val, doc, jointPath); ++childNum; @@ -104,9 +106,9 @@ private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, S processDocChildren(sdoc, fullPath); } - private String generateChildUniqueId(String parentId, String childKey, int childNum) { + private String generateChildUniqueId(String parentId, String childKey, String childNum) { // combines parentId with the child's key and childNum. e.g. "10/footnote/1" - return String.join(PATH_SEP_CHAR, parentId, childKey, Integer.toString(childNum)); + return parentId + PATH_SEP_CHAR + childKey +PATH_SEP_CHAR + childNum; } private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { From 8a2648083c78fe57a9e890f11696974a7ac6d484 Mon Sep 17 00:00:00 2001 From: Moshe Date: Thu, 5 Jul 2018 08:02:56 +0300 Subject: [PATCH 25/34] SOLR-12441: improve id less child doc test --- .../apache/solr/update/TestNestedUpdateProcessor.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index 35124739b0e9..b4017c222d02 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -120,7 +120,7 @@ public void before() throws Exception { public void testDeeplyNestedURPGrandChild() throws Exception { indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*" + PATH_SEP_CHAR + "grandChild", + assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*" + PATH_SEP_CHAR + "grandChild" + PATH_SEP_CHAR + "*" + PATH_SEP_CHAR, "fl","*", "sort","id desc", "wt","json"), @@ -132,7 +132,7 @@ public void testDeeplyNestedURPChildren() throws Exception { final String[] childrenTests = {"/response/docs/[0]/id=='" + childrenIds[0] + "'", "/response/docs/[1]/id=='" + childrenIds[1] + "'"}; indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":children", + assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":children" + PATH_SEP_CHAR + "*" + PATH_SEP_CHAR, "fl","*", "sort","id asc", "wt","json"), @@ -141,10 +141,10 @@ public void testDeeplyNestedURPChildren() throws Exception { @Test public void testDeeplyNestedURPChildrenWoId() throws Exception { - final String parentId = "3"; + final String rootId = "1"; final String childKey = "grandChild"; - final String expectedId = parentId + PATH_SEP_CHAR + childKey + PATH_SEP_CHAR + "0"; - SolrInputDocument noIdChildren = sdoc("id", "1", "children", sdocs(sdoc("id", "2", "name_s", "Yaz"), sdoc("id", parentId, "name_s", "Jazz", childKey, sdoc("names_s", "Gaz")))); + final String expectedId = rootId + PATH_SEP_CHAR + "children" + PATH_SEP_CHAR + "1" + PATH_SEP_CHAR + childKey + PATH_SEP_CHAR + "0"; + SolrInputDocument noIdChildren = sdoc("id", rootId, "children", sdocs(sdoc("name_s", "Yaz"), sdoc("name_s", "Jazz", childKey, sdoc("names_s", "Gaz")))); UpdateRequestProcessor nestedUpdate = new NestedUpdateProcessorFactory().getInstance(req(), null, null); AddUpdateCommand cmd = new AddUpdateCommand(req()); cmd.solrDoc = noIdChildren; From 895ef6dd95d4b49ad1cc3938748085fc87f97f91 Mon Sep 17 00:00:00 2001 From: Moshe Date: Thu, 5 Jul 2018 08:05:21 +0300 Subject: [PATCH 26/34] SOLR-12441: add "NEST" prefix to nested internal fields var name --- .../core/src/java/org/apache/solr/schema/IndexSchema.java | 4 ++-- .../update/processor/NestedUpdateProcessorFactory.java | 8 ++++---- .../org/apache/solr/update/TestNestedUpdateProcessor.java | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index f124f0e7089d..2cb26c717821 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -107,8 +107,8 @@ public class IndexSchema { public static final String LUCENE_MATCH_VERSION_PARAM = "luceneMatchVersion"; public static final String MAX_CHARS = "maxChars"; public static final String NAME = "name"; - public static final String PARENT_FIELD_NAME = "_NEST_PARENT_"; - public static final String PATH_FIELD_NAME = "_NEST_PATH_"; + public static final String NEST_PARENT_FIELD_NAME = "_NEST_PARENT_"; + public static final String NEST_PATH_FIELD_NAME = "_NEST_PATH_"; public static final String REQUIRED = "required"; public static final String SCHEMA = "schema"; public static final String SIMILARITY = "similarity"; diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index a14959d1a70d..5b48ea623a2f 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -39,11 +39,11 @@ public UpdateRequestProcessor getInstance(SolrQueryRequest req, SolrQueryRespons } private static boolean shouldStoreDocParent(IndexSchema schema) { - return schema.getFields().containsKey(IndexSchema.PARENT_FIELD_NAME); + return schema.getFields().containsKey(IndexSchema.NEST_PARENT_FIELD_NAME); } private static boolean shouldStoreDocPath(IndexSchema schema) { - return schema.getFields().containsKey(IndexSchema.PATH_FIELD_NAME); + return schema.getFields().containsKey(IndexSchema.NEST_PATH_FIELD_NAME); } } @@ -112,11 +112,11 @@ private String generateChildUniqueId(String parentId, String childKey, String ch } private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { - sdoc.setField(IndexSchema.PARENT_FIELD_NAME, parent.getFieldValue(uniqueKeyFieldName)); + sdoc.setField(IndexSchema.NEST_PARENT_FIELD_NAME, parent.getFieldValue(uniqueKeyFieldName)); } private void setPathField(SolrInputDocument sdoc, String fullPath) { - sdoc.setField(IndexSchema.PATH_FIELD_NAME, fullPath); + sdoc.setField(IndexSchema.NEST_PATH_FIELD_NAME, fullPath); } } diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index b4017c222d02..ae31ded71646 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -120,7 +120,7 @@ public void before() throws Exception { public void testDeeplyNestedURPGrandChild() throws Exception { indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":*" + PATH_SEP_CHAR + "grandChild" + PATH_SEP_CHAR + "*" + PATH_SEP_CHAR, + assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":*" + PATH_SEP_CHAR + "grandChild" + PATH_SEP_CHAR + "*" + PATH_SEP_CHAR, "fl","*", "sort","id desc", "wt","json"), @@ -132,7 +132,7 @@ public void testDeeplyNestedURPChildren() throws Exception { final String[] childrenTests = {"/response/docs/[0]/id=='" + childrenIds[0] + "'", "/response/docs/[1]/id=='" + childrenIds[1] + "'"}; indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.PATH_FIELD_NAME + ":children" + PATH_SEP_CHAR + "*" + PATH_SEP_CHAR, + assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":children" + PATH_SEP_CHAR + "*" + PATH_SEP_CHAR, "fl","*", "sort","id asc", "wt","json"), From c1662366f20eedd54f87c3c747300796556f2d25 Mon Sep 17 00:00:00 2001 From: Moshe Date: Thu, 5 Jul 2018 08:47:00 +0300 Subject: [PATCH 27/34] SOLR-12441: add num sep char to nested URP --- .../update/processor/NestedUpdateProcessorFactory.java | 5 +++-- .../org/apache/solr/update/TestNestedUpdateProcessor.java | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 5b48ea623a2f..1d82e3df1290 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -49,6 +49,7 @@ private static boolean shouldStoreDocPath(IndexSchema schema) { class NestedUpdateProcessor extends UpdateRequestProcessor { private static final String PATH_SEP_CHAR = "/"; + private static final String NUM_SEP_CHAR = ","; private boolean storePath; private boolean storeParent; private String uniqueKeyFieldName; @@ -83,7 +84,7 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); } final String sChildNum = String.valueOf(childNum); - final String lastPath = fieldName + PATH_SEP_CHAR + sChildNum + PATH_SEP_CHAR; + final String lastPath = fieldName + NUM_SEP_CHAR + sChildNum + NUM_SEP_CHAR; final String jointPath = fullPath == null ? lastPath : fullPath + PATH_SEP_CHAR + lastPath; SolrInputDocument cDoc = (SolrInputDocument) val; if(!cDoc.containsKey(uniqueKeyFieldName)) { @@ -108,7 +109,7 @@ private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, S private String generateChildUniqueId(String parentId, String childKey, String childNum) { // combines parentId with the child's key and childNum. e.g. "10/footnote/1" - return parentId + PATH_SEP_CHAR + childKey +PATH_SEP_CHAR + childNum; + return parentId + PATH_SEP_CHAR + childKey + NUM_SEP_CHAR + childNum; } private void setParentKey(SolrInputDocument sdoc, SolrInputDocument parent) { diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index ae31ded71646..fdbb4bb1571d 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -34,6 +34,7 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { private static final char PATH_SEP_CHAR = '/'; + private static final char NUM_SEP_CHAR = ','; private static final String[] childrenIds = { "2", "3" }; private static final String grandChildId = "4"; private static final String jDoc = "{\n" + @@ -120,7 +121,7 @@ public void before() throws Exception { public void testDeeplyNestedURPGrandChild() throws Exception { indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":*" + PATH_SEP_CHAR + "grandChild" + PATH_SEP_CHAR + "*" + PATH_SEP_CHAR, + assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":*" + PATH_SEP_CHAR + "grandChild" + NUM_SEP_CHAR + "*" + NUM_SEP_CHAR, "fl","*", "sort","id desc", "wt","json"), @@ -132,7 +133,7 @@ public void testDeeplyNestedURPChildren() throws Exception { final String[] childrenTests = {"/response/docs/[0]/id=='" + childrenIds[0] + "'", "/response/docs/[1]/id=='" + childrenIds[1] + "'"}; indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":children" + PATH_SEP_CHAR + "*" + PATH_SEP_CHAR, + assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":children" + NUM_SEP_CHAR + "*" + NUM_SEP_CHAR, "fl","*", "sort","id asc", "wt","json"), @@ -143,7 +144,7 @@ public void testDeeplyNestedURPChildren() throws Exception { public void testDeeplyNestedURPChildrenWoId() throws Exception { final String rootId = "1"; final String childKey = "grandChild"; - final String expectedId = rootId + PATH_SEP_CHAR + "children" + PATH_SEP_CHAR + "1" + PATH_SEP_CHAR + childKey + PATH_SEP_CHAR + "0"; + final String expectedId = rootId + PATH_SEP_CHAR + "children" + NUM_SEP_CHAR + "1" + PATH_SEP_CHAR + childKey + NUM_SEP_CHAR + "0"; SolrInputDocument noIdChildren = sdoc("id", rootId, "children", sdocs(sdoc("name_s", "Yaz"), sdoc("name_s", "Jazz", childKey, sdoc("names_s", "Gaz")))); UpdateRequestProcessor nestedUpdate = new NestedUpdateProcessorFactory().getInstance(req(), null, null); AddUpdateCommand cmd = new AddUpdateCommand(req()); From a75850069aa1e907ec8709268a6eaa7faffcc1c6 Mon Sep 17 00:00:00 2001 From: Moshe Date: Thu, 5 Jul 2018 12:22:56 +0300 Subject: [PATCH 28/34] SOLR-12441: add single val char to nested URP --- .../solr/update/processor/NestedUpdateProcessorFactory.java | 5 ++++- .../org/apache/solr/update/TestNestedUpdateProcessor.java | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 1d82e3df1290..875fc2dd7798 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -18,6 +18,7 @@ package org.apache.solr.update.processor; import java.io.IOException; +import java.util.Collection; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; @@ -50,6 +51,7 @@ private static boolean shouldStoreDocPath(IndexSchema schema) { class NestedUpdateProcessor extends UpdateRequestProcessor { private static final String PATH_SEP_CHAR = "/"; private static final String NUM_SEP_CHAR = ","; + private static final String SINGLE_VALUE_CHAR = "s"; private boolean storePath; private boolean storeParent; private String uniqueKeyFieldName; @@ -72,6 +74,7 @@ public void processAdd(AddUpdateCommand cmd) throws IOException { private void processDocChildren(SolrInputDocument doc, String fullPath) { int childNum = 0; for(SolrInputField field: doc.values()) { + boolean isSingleVal = !(field.getValue() instanceof Collection); for(Object val: field) { if(!(val instanceof SolrInputDocument)) { // either all collection items are child docs or none are. @@ -83,7 +86,7 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: '" + fieldName + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); } - final String sChildNum = String.valueOf(childNum); + final String sChildNum = isSingleVal ? SINGLE_VALUE_CHAR: String.valueOf(childNum); final String lastPath = fieldName + NUM_SEP_CHAR + sChildNum + NUM_SEP_CHAR; final String jointPath = fullPath == null ? lastPath : fullPath + PATH_SEP_CHAR + lastPath; SolrInputDocument cDoc = (SolrInputDocument) val; diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index fdbb4bb1571d..3de4fe70818e 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -35,6 +35,7 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { private static final char PATH_SEP_CHAR = '/'; private static final char NUM_SEP_CHAR = ','; + private static final String SINGLE_VAL_CHAR = "s"; private static final String[] childrenIds = { "2", "3" }; private static final String grandChildId = "4"; private static final String jDoc = "{\n" + @@ -144,7 +145,7 @@ public void testDeeplyNestedURPChildren() throws Exception { public void testDeeplyNestedURPChildrenWoId() throws Exception { final String rootId = "1"; final String childKey = "grandChild"; - final String expectedId = rootId + PATH_SEP_CHAR + "children" + NUM_SEP_CHAR + "1" + PATH_SEP_CHAR + childKey + NUM_SEP_CHAR + "0"; + final String expectedId = rootId + PATH_SEP_CHAR + "children" + NUM_SEP_CHAR + "1" + PATH_SEP_CHAR + childKey + NUM_SEP_CHAR + SINGLE_VAL_CHAR; SolrInputDocument noIdChildren = sdoc("id", rootId, "children", sdocs(sdoc("name_s", "Yaz"), sdoc("name_s", "Jazz", childKey, sdoc("names_s", "Gaz")))); UpdateRequestProcessor nestedUpdate = new NestedUpdateProcessorFactory().getInstance(req(), null, null); AddUpdateCommand cmd = new AddUpdateCommand(req()); From c8da19a0591124d575f30212076899c32d8db8b2 Mon Sep 17 00:00:00 2001 From: Moshe Date: Sun, 8 Jul 2018 08:59:56 +0300 Subject: [PATCH 29/34] SOLR-12411: change NUM_SEP_CHAR to # and SINGULAR_VALUE_CHAR to empty string --- .../update/processor/NestedUpdateProcessorFactory.java | 8 ++++---- .../org/apache/solr/update/TestNestedUpdateProcessor.java | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 875fc2dd7798..7912865b49d1 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -50,8 +50,8 @@ private static boolean shouldStoreDocPath(IndexSchema schema) { class NestedUpdateProcessor extends UpdateRequestProcessor { private static final String PATH_SEP_CHAR = "/"; - private static final String NUM_SEP_CHAR = ","; - private static final String SINGLE_VALUE_CHAR = "s"; + private static final String NUM_SEP_CHAR = "#"; + private static final String SINGULAR_VALUE_CHAR = " "; private boolean storePath; private boolean storeParent; private String uniqueKeyFieldName; @@ -86,7 +86,7 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Field name: '" + fieldName + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); } - final String sChildNum = isSingleVal ? SINGLE_VALUE_CHAR: String.valueOf(childNum); + final String sChildNum = isSingleVal ? SINGULAR_VALUE_CHAR : String.valueOf(childNum); final String lastPath = fieldName + NUM_SEP_CHAR + sChildNum + NUM_SEP_CHAR; final String jointPath = fullPath == null ? lastPath : fullPath + PATH_SEP_CHAR + lastPath; SolrInputDocument cDoc = (SolrInputDocument) val; @@ -111,7 +111,7 @@ private void processChildDoc(SolrInputDocument sdoc, SolrInputDocument parent, S } private String generateChildUniqueId(String parentId, String childKey, String childNum) { - // combines parentId with the child's key and childNum. e.g. "10/footnote/1" + // combines parentId with the child's key and childNum. e.g. "10/footnote#1" return parentId + PATH_SEP_CHAR + childKey + NUM_SEP_CHAR + childNum; } diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index 3de4fe70818e..2bb20417a0f4 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -34,8 +34,8 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { private static final char PATH_SEP_CHAR = '/'; - private static final char NUM_SEP_CHAR = ','; - private static final String SINGLE_VAL_CHAR = "s"; + private static final char NUM_SEP_CHAR = '#'; + private static final String SINGLE_VAL_CHAR = " "; private static final String[] childrenIds = { "2", "3" }; private static final String grandChildId = "4"; private static final String jDoc = "{\n" + From d399df75d7a3efac63093d4806c6ce2ba98adad0 Mon Sep 17 00:00:00 2001 From: Moshe Date: Sun, 8 Jul 2018 09:01:13 +0300 Subject: [PATCH 30/34] SOLR-12411: define lastPath & jointPath later closer to when they are used --- .../solr/update/processor/NestedUpdateProcessorFactory.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 7912865b49d1..6382c4288c83 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -87,13 +87,13 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { + "' contains: '" + PATH_SEP_CHAR + "' , which is reserved for the nested URP"); } final String sChildNum = isSingleVal ? SINGULAR_VALUE_CHAR : String.valueOf(childNum); - final String lastPath = fieldName + NUM_SEP_CHAR + sChildNum + NUM_SEP_CHAR; - final String jointPath = fullPath == null ? lastPath : fullPath + PATH_SEP_CHAR + lastPath; SolrInputDocument cDoc = (SolrInputDocument) val; if(!cDoc.containsKey(uniqueKeyFieldName)) { String parentDocId = doc.getField(uniqueKeyFieldName).getFirstValue().toString(); cDoc.setField(uniqueKeyFieldName, generateChildUniqueId(parentDocId, fieldName, sChildNum)); } + final String lastPath = fieldName + NUM_SEP_CHAR + sChildNum + NUM_SEP_CHAR; + final String jointPath = fullPath == null ? lastPath : fullPath + PATH_SEP_CHAR + lastPath; processChildDoc((SolrInputDocument) val, doc, jointPath); ++childNum; } From 16d7c10a4e9e77daea28f4856dc6ad05d4a89d6d Mon Sep 17 00:00:00 2001 From: Moshe Date: Sun, 8 Jul 2018 09:04:03 +0300 Subject: [PATCH 31/34] SOLR-12411: fix counting of child index --- .../solr/update/processor/NestedUpdateProcessorFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 6382c4288c83..525155072633 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -72,8 +72,8 @@ public void processAdd(AddUpdateCommand cmd) throws IOException { } private void processDocChildren(SolrInputDocument doc, String fullPath) { - int childNum = 0; for(SolrInputField field: doc.values()) { + int childNum = 0; boolean isSingleVal = !(field.getValue() instanceof Collection); for(Object val: field) { if(!(val instanceof SolrInputDocument)) { From dec76e23d30614a7e860a8b183c7a2e4f857152d Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 9 Jul 2018 16:53:22 +0300 Subject: [PATCH 32/34] SOLR-12441: only add one NUM_SEP_CHAR for _NEST_PATH_ in NestedUpdateProcessorFactory --- .../update/processor/NestedUpdateProcessorFactory.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java index 525155072633..a3c7dcef6730 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/NestedUpdateProcessorFactory.java @@ -51,7 +51,7 @@ private static boolean shouldStoreDocPath(IndexSchema schema) { class NestedUpdateProcessor extends UpdateRequestProcessor { private static final String PATH_SEP_CHAR = "/"; private static final String NUM_SEP_CHAR = "#"; - private static final String SINGULAR_VALUE_CHAR = " "; + private static final String SINGULAR_VALUE_CHAR = ""; private boolean storePath; private boolean storeParent; private String uniqueKeyFieldName; @@ -92,9 +92,10 @@ private void processDocChildren(SolrInputDocument doc, String fullPath) { String parentDocId = doc.getField(uniqueKeyFieldName).getFirstValue().toString(); cDoc.setField(uniqueKeyFieldName, generateChildUniqueId(parentDocId, fieldName, sChildNum)); } - final String lastPath = fieldName + NUM_SEP_CHAR + sChildNum + NUM_SEP_CHAR; - final String jointPath = fullPath == null ? lastPath : fullPath + PATH_SEP_CHAR + lastPath; - processChildDoc((SolrInputDocument) val, doc, jointPath); + final String lastKeyPath = fieldName + NUM_SEP_CHAR + sChildNum; + // concat of all paths children.grandChild => children#1/grandChild# + final String childDocPath = fullPath == null ? lastKeyPath : fullPath + PATH_SEP_CHAR + lastKeyPath; + processChildDoc((SolrInputDocument) val, doc, childDocPath); ++childNum; } } From bf53e6ec8f228792c017624d7eaed4ffa4209511 Mon Sep 17 00:00:00 2001 From: Moshe Date: Mon, 9 Jul 2018 16:54:58 +0300 Subject: [PATCH 33/34] SOLR-12441: add tests for _NEST_PATH_FIELD --- .../update/TestNestedUpdateProcessor.java | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index 2bb20417a0f4..71aed05a0a9c 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -35,9 +35,10 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { private static final char PATH_SEP_CHAR = '/'; private static final char NUM_SEP_CHAR = '#'; - private static final String SINGLE_VAL_CHAR = " "; + private static final String SINGLE_VAL_CHAR = ""; private static final String[] childrenIds = { "2", "3" }; private static final String grandChildId = "4"; + private static final String secondChildList = "anotherChildList"; private static final String jDoc = "{\n" + " \"add\": {\n" + " \"doc\": {\n" + @@ -57,6 +58,7 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { " \"foo_s\": \"Bar\"\n" + " }\n" + " ]\n" + + secondChildList + ": [{\"id\": \"4\", \"last_s\": \"Smith\"}],\n" + " }\n" + " }\n" + "}"; @@ -120,25 +122,41 @@ public void before() throws Exception { @Test public void testDeeplyNestedURPGrandChild() throws Exception { + final String[] tests = { + "/response/docs/[0]/id=='" + grandChildId + "'", + "/response/docs/[0]/" + IndexSchema.NEST_PATH_FIELD_NAME + "=='children#0/grandChild#'" + }; indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":*" + PATH_SEP_CHAR + "grandChild" + NUM_SEP_CHAR + "*" + NUM_SEP_CHAR, + assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":*" + PATH_SEP_CHAR + "grandChild" + NUM_SEP_CHAR + "*", "fl","*", "sort","id desc", "wt","json"), - "/response/docs/[0]/id=='" + grandChildId + "'"); + tests); } @Test public void testDeeplyNestedURPChildren() throws Exception { - final String[] childrenTests = {"/response/docs/[0]/id=='" + childrenIds[0] + "'", "/response/docs/[1]/id=='" + childrenIds[1] + "'"}; + final String[] childrenTests = { + "/response/docs/[0]/id=='" + childrenIds[0] + "'", + "/response/docs/[1]/id=='" + childrenIds[1] + "'", + "/response/docs/[0]/" + IndexSchema.NEST_PATH_FIELD_NAME + "=='children#0'", + "/response/docs/[1]/" + IndexSchema.NEST_PATH_FIELD_NAME + "=='children#1'" + }; indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":children" + NUM_SEP_CHAR + "*" + NUM_SEP_CHAR, + assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":children" + NUM_SEP_CHAR + "?", "fl","*", "sort","id asc", "wt","json"), childrenTests); + + assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":" + secondChildList + NUM_SEP_CHAR + "?", + "fl","*", + "sort","id asc", + "wt","json"), + "/response/docs/[0]/id=='4'", + "/response/docs/[0]/" + IndexSchema.NEST_PATH_FIELD_NAME + "=='" + secondChildList + "#0'"); } @Test From 259acf79ae792ffbfe1fbdc46f4526acae44566d Mon Sep 17 00:00:00 2001 From: Moshe Date: Tue, 10 Jul 2018 08:41:47 +0300 Subject: [PATCH 34/34] SOLR-12441: add sanity unit test TestNestedUpdateProcessor#testDeeplyNestedURPSanity --- .../update/TestNestedUpdateProcessor.java | 44 ++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java index 71aed05a0a9c..b71a9c569907 100644 --- a/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java +++ b/solr/core/src/test/org/apache/solr/update/TestNestedUpdateProcessor.java @@ -36,7 +36,6 @@ public class TestNestedUpdateProcessor extends SolrTestCaseJ4 { private static final char PATH_SEP_CHAR = '/'; private static final char NUM_SEP_CHAR = '#'; private static final String SINGLE_VAL_CHAR = ""; - private static final String[] childrenIds = { "2", "3" }; private static final String grandChildId = "4"; private static final String secondChildList = "anotherChildList"; private static final String jDoc = "{\n" + @@ -123,12 +122,12 @@ public void before() throws Exception { @Test public void testDeeplyNestedURPGrandChild() throws Exception { final String[] tests = { - "/response/docs/[0]/id=='" + grandChildId + "'", + "/response/docs/[0]/id=='4'", "/response/docs/[0]/" + IndexSchema.NEST_PATH_FIELD_NAME + "=='children#0/grandChild#'" }; indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":*" + PATH_SEP_CHAR + "grandChild" + NUM_SEP_CHAR + "*", + assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":*/grandChild#*", "fl","*", "sort","id desc", "wt","json"), @@ -138,33 +137,58 @@ public void testDeeplyNestedURPGrandChild() throws Exception { @Test public void testDeeplyNestedURPChildren() throws Exception { final String[] childrenTests = { - "/response/docs/[0]/id=='" + childrenIds[0] + "'", - "/response/docs/[1]/id=='" + childrenIds[1] + "'", + "/response/docs/[0]/id=='2'", + "/response/docs/[1]/id=='3'", "/response/docs/[0]/" + IndexSchema.NEST_PATH_FIELD_NAME + "=='children#0'", "/response/docs/[1]/" + IndexSchema.NEST_PATH_FIELD_NAME + "=='children#1'" }; indexSampleData(jDoc); - assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":children" + NUM_SEP_CHAR + "?", + assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":children#?", "fl","*", "sort","id asc", "wt","json"), childrenTests); - assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":" + secondChildList + NUM_SEP_CHAR + "?", + assertJQ(req("q", IndexSchema.NEST_PATH_FIELD_NAME + ":anotherChildList#?", "fl","*", "sort","id asc", "wt","json"), "/response/docs/[0]/id=='4'", - "/response/docs/[0]/" + IndexSchema.NEST_PATH_FIELD_NAME + "=='" + secondChildList + "#0'"); + "/response/docs/[0]/" + IndexSchema.NEST_PATH_FIELD_NAME + "=='anotherChildList#0'"); + } + + @Test + public void testDeeplyNestedURPSanity() throws Exception { + SolrInputDocument docHierarchy = sdoc("id", "1", "children", sdocs(sdoc("id", "2", "name_s", "Yaz"), + sdoc("id", "3", "name_s", "Jazz", "grandChild", sdoc("id", "4", "name_s", "Gaz"))), "lonelyChild", sdoc("id", "5", "name_s", "Loner")); + UpdateRequestProcessor nestedUpdate = new NestedUpdateProcessorFactory().getInstance(req(), null, null); + AddUpdateCommand cmd = new AddUpdateCommand(req()); + cmd.solrDoc = docHierarchy; + nestedUpdate.processAdd(cmd); + cmd.clear(); + + List children = (List) docHierarchy.get("children").getValues(); + + SolrInputDocument firstChild = (SolrInputDocument) children.get(0); + assertEquals("SolrInputDocument(fields: [id=2, name_s=Yaz, _NEST_PATH_=children#0, _NEST_PARENT_=1])", firstChild.toString()); + + SolrInputDocument secondChild = (SolrInputDocument) children.get(1); + assertEquals("SolrInputDocument(fields: [id=3, name_s=Jazz, grandChild=SolrInputDocument(fields: [id=4, name_s=Gaz, _NEST_PATH_=children#1/grandChild#, _NEST_PARENT_=3]), _NEST_PATH_=children#1, _NEST_PARENT_=1])", secondChild.toString()); + + SolrInputDocument grandChild = (SolrInputDocument)((SolrInputDocument) children.get(1)).get("grandChild").getValue(); + assertEquals("SolrInputDocument(fields: [id=4, name_s=Gaz, _NEST_PATH_=children#1/grandChild#, _NEST_PARENT_=3])", grandChild.toString()); + + SolrInputDocument singularChild = (SolrInputDocument) docHierarchy.get("lonelyChild").getValue(); + assertEquals("SolrInputDocument(fields: [id=5, name_s=Loner, _NEST_PATH_=lonelyChild#, _NEST_PARENT_=1])", singularChild.toString()); } @Test public void testDeeplyNestedURPChildrenWoId() throws Exception { final String rootId = "1"; final String childKey = "grandChild"; - final String expectedId = rootId + PATH_SEP_CHAR + "children" + NUM_SEP_CHAR + "1" + PATH_SEP_CHAR + childKey + NUM_SEP_CHAR + SINGLE_VAL_CHAR; - SolrInputDocument noIdChildren = sdoc("id", rootId, "children", sdocs(sdoc("name_s", "Yaz"), sdoc("name_s", "Jazz", childKey, sdoc("names_s", "Gaz")))); + final String expectedId = rootId + "/children#1/" + childKey + NUM_SEP_CHAR + SINGLE_VAL_CHAR; + SolrInputDocument noIdChildren = sdoc("id", rootId, "children", sdocs(sdoc("name_s", "Yaz"), sdoc("name_s", "Jazz", childKey, sdoc("name_s", "Gaz")))); UpdateRequestProcessor nestedUpdate = new NestedUpdateProcessorFactory().getInstance(req(), null, null); AddUpdateCommand cmd = new AddUpdateCommand(req()); cmd.solrDoc = noIdChildren;