From 3603832e3d4fdb3bfe3fa9eb27202b71dda4e068 Mon Sep 17 00:00:00 2001 From: Robert Alexandersson Date: Thu, 16 Feb 2017 17:06:20 +0100 Subject: [PATCH 1/4] SOLR-10134 Added the httpMethod param to context so SolR SchemaAPI can handle POST calls --- .../apache/solr/client/solrj/embedded/EmbeddedSolrServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java index fc283f42ef61..761e8923eefa 100644 --- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java +++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java @@ -172,6 +172,7 @@ public NamedList request(SolrRequest request, String coreName) throws So req = _parser.buildRequestFrom(core, params, request.getContentStreams()); req.getContext().put(PATH, path); + req.getContext().put("httpMethod", request.getMethod().name()); SolrQueryResponse rsp = new SolrQueryResponse(); SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp)); From 6de410703d0e1be497289017ca1e1c1c4d8303bd Mon Sep 17 00:00:00 2001 From: Robert Alexandersson Date: Fri, 17 Feb 2017 08:54:43 +0100 Subject: [PATCH 2/4] SOLR-10134 Added testcase and enabled the SchemaHandler to parse the incoming command --- .../solrj/embedded/EmbeddedSolrServer.java | 54 ++++++----- .../solr/request/SolrQueryRequestBase.java | 18 +++- .../solr/servlet/SolrRequestParsers.java | 4 +- .../TestEmbeddedSolrServerSchemaAPI.java | 89 +++++++++++++++++++ .../java/org/apache/solr/SolrTestCaseJ4.java | 27 ++++++ 5 files changed, 165 insertions(+), 27 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java index 761e8923eefa..8de5fc92fafa 100644 --- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java +++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java @@ -200,32 +200,13 @@ public void writeResults(ResultContext ctx, JavaBinCodec codec) throws IOExcepti }; - ByteArrayOutputStream out = new ByteArrayOutputStream(); - new JavaBinCodec(resolver) { + try(ByteArrayOutputStream out = new ByteArrayOutputStream()) { + createJavaBinCodec(callback, resolver).setWritableDocFields(resolver).marshal(rsp.getValues(), out); - @Override - public void writeSolrDocument(SolrDocument doc) { - callback.streamSolrDocument(doc); - //super.writeSolrDocument( doc, fields ); + try(InputStream in = out.toInputStream()){ + return (NamedList) new JavaBinCodec(resolver).unmarshal(in); } - - @Override - public void writeSolrDocumentList(SolrDocumentList docs) throws IOException { - if (docs.size() > 0) { - SolrDocumentList tmp = new SolrDocumentList(); - tmp.setMaxScore(docs.getMaxScore()); - tmp.setNumFound(docs.getNumFound()); - tmp.setStart(docs.getStart()); - docs = tmp; - } - callback.streamDocListInfo(docs.getNumFound(), docs.getStart(), docs.getMaxScore()); - super.writeSolrDocumentList(docs); - } - - }.setWritableDocFields(resolver). marshal(rsp.getValues(), out); - - InputStream in = out.toInputStream(); - return (NamedList) new JavaBinCodec(resolver).unmarshal(in); + } } catch (Exception ex) { throw new RuntimeException(ex); } @@ -244,6 +225,31 @@ public void writeSolrDocumentList(SolrDocumentList docs) throws IOException { } } + private JavaBinCodec createJavaBinCodec(final StreamingResponseCallback callback, final BinaryResponseWriter.Resolver resolver) { + return new JavaBinCodec(resolver) { + + @Override + public void writeSolrDocument(SolrDocument doc) { + callback.streamSolrDocument(doc); + //super.writeSolrDocument( doc, fields ); + } + + @Override + public void writeSolrDocumentList(SolrDocumentList docs) throws IOException { + if (docs.size() > 0) { + SolrDocumentList tmp = new SolrDocumentList(); + tmp.setMaxScore(docs.getMaxScore()); + tmp.setNumFound(docs.getNumFound()); + tmp.setStart(docs.getStart()); + docs = tmp; + } + callback.streamDocListInfo(docs.getNumFound(), docs.getStart(), docs.getMaxScore()); + super.writeSolrDocumentList(docs); + } + + }; + } + private static void checkForExceptions(SolrQueryResponse rsp) throws Exception { if (rsp.getException() != null) { if (rsp.getException() instanceof SolrException) { diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java index 4b0e4d62d8ba..afbc92ac3674 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java +++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java @@ -18,6 +18,7 @@ import org.apache.solr.api.ApiBag; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.common.util.ValidatingJsonMap; import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.search.SolrIndexSearcher; @@ -31,6 +32,7 @@ import org.apache.solr.core.SolrCore; import java.io.Closeable; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.security.Principal; @@ -202,7 +204,7 @@ public List getCommands(boolean validateInput) { Iterable contentStreams = getContentStreams(); if (contentStreams == null) throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No content stream"); for (ContentStream contentStream : contentStreams) { - parsedCommands = ApiBag.getCommandOperations(new InputStreamReader((InputStream) contentStream, UTF_8), + parsedCommands = ApiBag.getCommandOperations(getInputStream(contentStream), getValidators(), validateInput); } @@ -211,6 +213,20 @@ public List getCommands(boolean validateInput) { } + private InputStreamReader getInputStream(ContentStream contentStream) { + if(contentStream instanceof InputStream) { + return new InputStreamReader((InputStream)contentStream, UTF_8); + } + if(contentStream instanceof ContentStreamBase.StringStream) { + try { + return new InputStreamReader(contentStream.getStream(), UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + throw new IllegalAccessError("ContentStream type not supported "+contentStream.getClass().getName()); + } + protected ValidatingJsonMap getSpec() { return null; } diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java index c311d4aad4c1..d2067b53d5d2 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java @@ -239,7 +239,7 @@ public List getCommands(boolean validateInput) { if (httpSolrCall != null) { return httpSolrCall.getCommands(validateInput); } - return Collections.emptyList(); + return super.getCommands(validateInput); } @Override @@ -247,7 +247,7 @@ public Map getPathTemplateValues() { if (httpSolrCall != null && httpSolrCall instanceof V2HttpCall) { return ((V2HttpCall) httpSolrCall).getUrlParts(); } - return Collections.EMPTY_MAP; + return super.getPathTemplateValues(); } @Override diff --git a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java new file mode 100644 index 000000000000..fc6c7ce77d57 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.embedded; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.request.schema.SchemaRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.schema.SchemaResponse; +import org.apache.solr.core.NodeConfig; +import org.apache.solr.core.SolrResourceLoader; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestEmbeddedSolrServerSchemaAPI extends SolrTestCaseJ4 { + + + @BeforeClass + public static void initClass() { + System.setProperty("managed.schema.mutable", "true"); + } + + @AfterClass + public static void destroyClass() { + System.clearProperty("managed.schema.mutable"); + } + + @Test + public void testNodeConfigConstructor() throws Exception { + Path path = createTempDir(); + + SolrResourceLoader loader = new SolrResourceLoader(path); + NodeConfig config = new NodeConfig.NodeConfigBuilder("testnode", loader) + .setConfigSetBaseDirectory(Paths.get(TEST_HOME()).resolve("configsets").toString()) + .build(); + + try (EmbeddedSolrServer server = new EmbeddedSolrServer(config, "collection1")) { + + + CoreAdminRequest.Create createRequest = new CoreAdminRequest.Create(); + createRequest.setCoreName("collection1"); + createRequest.setConfigSet("cloud-managed"); + CoreAdminResponse coreAdminResponse = createRequest.process(server); + + SolrTestCaseJ4.assertResponse(coreAdminResponse); + + Map fieldAttributes = new LinkedHashMap<>(); + String fieldName = "VerificationTest"; + fieldAttributes.put("name", fieldName); + fieldAttributes.put("type", "string"); + fieldAttributes.put("stored", false); + fieldAttributes.put("indexed", true); + fieldAttributes.put("multiValued", true); + SchemaRequest.AddField addFieldUpdateSchemaRequest = new SchemaRequest.AddField(fieldAttributes); + SchemaResponse.UpdateResponse addFieldResponse = addFieldUpdateSchemaRequest.process(server); + + SolrTestCaseJ4.assertResponse(addFieldResponse); + + + // This asserts that the field was actually created + SchemaRequest.Field fieldRequest = new SchemaRequest.Field(fieldName); + Map foundFieldAttributes = fieldRequest.process(server).getField(); + + assertEquals(fieldAttributes.get("name"), foundFieldAttributes.get("name")); + + } + } + +} diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 1dcd529354e5..98270b6973e4 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -78,6 +78,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.impl.LBHttpSolrClient; +import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.IpTables; import org.apache.solr.common.SolrDocument; @@ -2390,6 +2391,32 @@ public static HttpSolrClient getHttpSolrClient(String url) { .build(); } + public static void assertResponse(SolrResponseBase response) { + assertTrue(response.getStatus() == 0); + if (response.getResponse().get("errors") != null) { + try { + + String message; + Object errorMessages = ((LinkedHashMap)((ArrayList) response.getResponse().get("errors")).get(0)).get("errorMessages"); + if(errorMessages instanceof String) { + message = errorMessages.toString(); + }else{ + message = String.valueOf( + ((ArrayList) + ((LinkedHashMap) + ((ArrayList) response.getResponse().get("errors")) + .get(0)) + .get("errorMessages")) + .get(0)); + } + assertTrue(message, response.getResponse().get("errors") == null); + } catch (Exception e) { + assertTrue("Generic error, found errors but unable to find message", + response.getResponse().get("errors") == null); + } + } + } + /** * Returns a randomly generated Date in the appropriate Solr external (input) format * @see #randomSkewedDate From 09559122d445aa4efa4cf9beffa6622120fe5f6f Mon Sep 17 00:00:00 2001 From: Robert Alexandersson Date: Fri, 17 Feb 2017 08:59:25 +0100 Subject: [PATCH 3/4] SOLR-10134 Added testcase and enabled the SchemaHandler to parse the incoming command --- .../java/org/apache/solr/request/SolrQueryRequestBase.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java index afbc92ac3674..37aa30b54fd0 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java +++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java @@ -216,15 +216,13 @@ public List getCommands(boolean validateInput) { private InputStreamReader getInputStream(ContentStream contentStream) { if(contentStream instanceof InputStream) { return new InputStreamReader((InputStream)contentStream, UTF_8); - } - if(contentStream instanceof ContentStreamBase.StringStream) { + } else { try { return new InputStreamReader(contentStream.getStream(), UTF_8); } catch (IOException e) { throw new RuntimeException(e); } } - throw new IllegalAccessError("ContentStream type not supported "+contentStream.getClass().getName()); } protected ValidatingJsonMap getSpec() { From dc02bcf8de62169dbbe9674c5d5f899c5064e37c Mon Sep 17 00:00:00 2001 From: Robert Alexandersson Date: Fri, 17 Feb 2017 10:03:00 +0100 Subject: [PATCH 4/4] SOLR-10134 Improved the test case and added a failing test for immutable config --- .../TestEmbeddedSolrServerSchemaAPI.java | 63 ++++++++++++++++++- .../java/org/apache/solr/SolrTestCaseJ4.java | 33 +++++----- 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java index fc6c7ce77d57..6b799b4b7246 100644 --- a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java +++ b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerSchemaAPI.java @@ -16,11 +16,13 @@ */ package org.apache.solr.client.solrj.embedded; +import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; import java.util.LinkedHashMap; import java.util.Map; +import org.apache.commons.io.FileUtils; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.schema.SchemaRequest; @@ -32,6 +34,7 @@ import org.junit.BeforeClass; import org.junit.Test; + public class TestEmbeddedSolrServerSchemaAPI extends SolrTestCaseJ4 { @@ -46,7 +49,7 @@ public static void destroyClass() { } @Test - public void testNodeConfigConstructor() throws Exception { + public void testSchemaAddFieldAndVerifyExistence() throws Exception { Path path = createTempDir(); SolrResourceLoader loader = new SolrResourceLoader(path); @@ -54,12 +57,21 @@ public void testNodeConfigConstructor() throws Exception { .setConfigSetBaseDirectory(Paths.get(TEST_HOME()).resolve("configsets").toString()) .build(); - try (EmbeddedSolrServer server = new EmbeddedSolrServer(config, "collection1")) { + String configSet = "cloud-managed"; + + // Backup the schema config as its not reset on reruns of the tests + File schemaFile = new File(config.getConfigSetBaseDirectory().toFile(), configSet+"/conf/managed-schema"); + File schemaFileBackup = new File(config.getConfigSetBaseDirectory().toFile(), configSet+"/conf/managed-schema.backup"); + assertTrue("The schema intended for backup was not found", schemaFile.exists()); + FileUtils.copyFile(schemaFile, schemaFileBackup); + assertTrue("The schema backup was not found", schemaFileBackup.exists()); + + try (EmbeddedSolrServer server = new EmbeddedSolrServer(config, "collection1")) { CoreAdminRequest.Create createRequest = new CoreAdminRequest.Create(); createRequest.setCoreName("collection1"); - createRequest.setConfigSet("cloud-managed"); + createRequest.setConfigSet(configSet); CoreAdminResponse coreAdminResponse = createRequest.process(server); SolrTestCaseJ4.assertResponse(coreAdminResponse); @@ -78,10 +90,55 @@ public void testNodeConfigConstructor() throws Exception { // This asserts that the field was actually created + // this is due to the fact that the response gave OK but actually never created the field. SchemaRequest.Field fieldRequest = new SchemaRequest.Field(fieldName); Map foundFieldAttributes = fieldRequest.process(server).getField(); assertEquals(fieldAttributes.get("name"), foundFieldAttributes.get("name")); + assertEquals(fieldAttributes.get("type"), foundFieldAttributes.get("type")); + assertEquals(fieldAttributes.get("stored"), foundFieldAttributes.get("stored")); + assertEquals(fieldAttributes.get("indexed"), foundFieldAttributes.get("indexed")); + assertEquals(fieldAttributes.get("multiValued"), foundFieldAttributes.get("multiValued")); + + }finally { + // Restore the schema-file + FileUtils.copyFile(schemaFileBackup, schemaFile); + schemaFileBackup.deleteOnExit(); + + } + } + + @Test + public void testSchemaAddFieldAndFailOnImmutable() throws Exception { + Path path = createTempDir(); + + SolrResourceLoader loader = new SolrResourceLoader(path); + NodeConfig config = new NodeConfig.NodeConfigBuilder("testnode", loader) + .setConfigSetBaseDirectory(Paths.get(TEST_HOME()).resolve("configsets").toString()) + .build(); + + String configSet = "minimal"; + + try (EmbeddedSolrServer server = new EmbeddedSolrServer(config, "collection1")) { + + CoreAdminRequest.Create createRequest = new CoreAdminRequest.Create(); + createRequest.setCoreName("collection1"); + createRequest.setConfigSet(configSet); + CoreAdminResponse coreAdminResponse = createRequest.process(server); + + SolrTestCaseJ4.assertResponse(coreAdminResponse); + + Map fieldAttributes = new LinkedHashMap<>(); + String fieldName = "VerificationTest"; + fieldAttributes.put("name", fieldName); + fieldAttributes.put("type", "string"); + fieldAttributes.put("stored", false); + fieldAttributes.put("indexed", true); + fieldAttributes.put("multiValued", true); + SchemaRequest.AddField addFieldUpdateSchemaRequest = new SchemaRequest.AddField(fieldAttributes); + SchemaResponse.UpdateResponse addFieldResponse = addFieldUpdateSchemaRequest.process(server); + + assertEquals("schema is not editable", SolrTestCaseJ4.getResponseMessage(addFieldResponse)); } } diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 98270b6973e4..76a12f4e6a3d 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -2395,21 +2395,7 @@ public static void assertResponse(SolrResponseBase response) { assertTrue(response.getStatus() == 0); if (response.getResponse().get("errors") != null) { try { - - String message; - Object errorMessages = ((LinkedHashMap)((ArrayList) response.getResponse().get("errors")).get(0)).get("errorMessages"); - if(errorMessages instanceof String) { - message = errorMessages.toString(); - }else{ - message = String.valueOf( - ((ArrayList) - ((LinkedHashMap) - ((ArrayList) response.getResponse().get("errors")) - .get(0)) - .get("errorMessages")) - .get(0)); - } - assertTrue(message, response.getResponse().get("errors") == null); + assertTrue("Assertion failure on response, message: "+ getResponseMessage(response), response.getResponse().get("errors") == null); } catch (Exception e) { assertTrue("Generic error, found errors but unable to find message", response.getResponse().get("errors") == null); @@ -2417,6 +2403,23 @@ public static void assertResponse(SolrResponseBase response) { } } + public static String getResponseMessage(SolrResponseBase response) { + String message; + Object errorMessages = ((LinkedHashMap)((ArrayList) response.getResponse().get("errors")).get(0)).get("errorMessages"); + if(errorMessages instanceof String) { + message = errorMessages.toString(); + }else{ + message = String.valueOf( + ((ArrayList) + ((LinkedHashMap) + ((ArrayList) response.getResponse().get("errors")) + .get(0)) + .get("errorMessages")) + .get(0)); + } + return message; + } + /** * Returns a randomly generated Date in the appropriate Solr external (input) format * @see #randomSkewedDate