diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java index 5e40c0d859b9..2cfdcb2a87db 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java @@ -74,10 +74,26 @@ * A {@link org.apache.solr.request.SolrRequestHandler} for ConfigSets API requests. */ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNameProvider { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final Boolean unitTest; + static { + Boolean unitTest1; + try { + Class.forName("org.apache.solr.cloud.SolrCloudTestCase"); + unitTest1 = true; + log.warn("{} thinks it is running in a unit test. If you see " + + "this message in a production server please check that you have not added SolrCloudTestCase classes " + + "to your server's classpath. If this class is not present, or is present in a default installation, " + + "please report as a bug.", ConfigSetsHandler.class.getCanonicalName()); + } catch (ClassNotFoundException ignored) { + unitTest1 = false; + } + unitTest = unitTest1; + } final public static Boolean DISABLE_CREATE_AUTH_CHECKS = Boolean.getBoolean("solr.disableConfigSetsCreateAuthChecks"); // this is for back compat only final public static String DEFAULT_CONFIGSET_NAME = "_default"; final public static String AUTOCREATED_CONFIGSET_SUFFIX = ".AUTOCREATED"; - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); protected final CoreContainer coreContainer; public static long DEFAULT_ZK_TIMEOUT = 300 * 1000; /** @@ -436,7 +452,7 @@ public Map call(SolrQueryRequest req, SolrQueryResponse rsp, Con Map props = CollectionsHandler.copy(req.getParams().required(), null, NAME); props.put(BASE_CONFIGSET, baseConfigSetName); - if (!DISABLE_CREATE_AUTH_CHECKS && + if (!getDisableCreateAuthChecks() && !isTrusted(req, h.coreContainer.getAuthenticationPlugin()) && isCurrentlyTrusted(h.coreContainer.getZkController().getZkClient(), ZkConfigManager.CONFIGS_ZKNODE + "/" + baseConfigSetName)) { throw new SolrException(ErrorCode.UNAUTHORIZED, "Can't create a configset with an unauthenticated request from a trusted " + BASE_CONFIGSET); @@ -481,6 +497,10 @@ public static ConfigSetOperation get(ConfigSetAction action) { } } + private static Boolean getDisableCreateAuthChecks() { + return unitTest ? Boolean.getBoolean("solr.disableConfigSetsCreateAuthChecks") : DISABLE_CREATE_AUTH_CHECKS; + } + @Override public Name getPermissionName(AuthorizationContext ctx) { String a = ctx.getParams().get(ConfigSetParams.ACTION); diff --git a/solr/core/src/test/org/apache/solr/update/processor/CategoryRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/CategoryRoutedAliasUpdateProcessorTest.java index 9e3d6b8af6f8..a7532754bb49 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/CategoryRoutedAliasUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/CategoryRoutedAliasUpdateProcessorTest.java @@ -42,6 +42,7 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,10 +65,7 @@ public class CategoryRoutedAliasUpdateProcessorTest extends RoutedAliasUpdatePro private int lastDocId = 0; private static CloudSolrClient solrClient; private int numDocsDeletedOrFailed = 0; - // uncomment to create pause for attaching profiler. -// static { -// JOptionPane.showMessageDialog(null,"Ready?"); -// } + private static final String origSysprop=System.getProperty("solr.disableConfigSetsCreateAuthChecks"); @Before public void doBefore() throws Exception { @@ -88,8 +86,18 @@ public void doAfter() throws Exception { } } + @BeforeClass + public static void setUpClass() { + System.setProperty("solr.disableConfigSetsCreateAuthChecks", "true"); + } + @AfterClass public static void cleanUpAfterClass() throws Exception { + if (origSysprop != null) { + System.setProperty("solr.disableConfigSetsCreateAuthChecks", origSysprop); + } else { + System.getProperties().remove("solr.disableConfigSetsCreateAuthChecks"); + } solrClient = null; } diff --git a/solr/core/src/test/org/apache/solr/update/processor/DimensionalRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/DimensionalRoutedAliasUpdateProcessorTest.java index ef516de42ce1..d43370f7c7e9 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/DimensionalRoutedAliasUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/DimensionalRoutedAliasUpdateProcessorTest.java @@ -45,6 +45,7 @@ import org.junit.After; import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +54,7 @@ import static org.apache.solr.client.solrj.request.CollectionAdminRequest.createTimeRoutedAlias; public class DimensionalRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcessorTest { + private static final String origSysprop = System.getProperty("solr.disableConfigSetsCreateAuthChecks"); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -84,8 +86,18 @@ public void doAfter() throws Exception { shutdownCluster(); } + @BeforeClass + public static void setUpClass() { + System.setProperty("solr.disableConfigSetsCreateAuthChecks", "true"); + } + @AfterClass - public static void finish() throws Exception { + public static void cleanUpAfterClass() throws Exception { + if (origSysprop != null) { + System.setProperty("solr.disableConfigSetsCreateAuthChecks", origSysprop); + } else { + System.getProperties().remove("solr.disableConfigSetsCreateAuthChecks"); + } IOUtils.close(solrClient); solrClient = null; } @@ -125,7 +137,8 @@ public void testTimeCat() throws Exception { ), ap( "tabby" - ) + ), + 1 ); addDocsAndCommit(true, newDoc("calico", "2019-07-02T00:00:00Z")); @@ -141,11 +154,12 @@ public void testTimeCat() throws Exception { ap( "tabby", "calico" - ) + ), + 2 ); testFailedDocument("shorthair", "2017-10-23T00:00:00Z", "couldn't be routed" ); - testFailedDocument("shorthair", "2020-10-23T00:00:00Z", "too far in the future" ); + testFailedDocument("shorthair", "2090-10-23T00:00:00Z", "too far in the future" ); testFailedDocument(null, "2019-07-02T00:00:00Z", "Route value is null"); testFailedDocument("foo__CRA__bar", "2019-07-02T00:00:00Z", "7 character sequence __CRA__"); testFailedDocument("fóóCRAóóbar", "2019-07-02T00:00:00Z", "7 character sequence __CRA__"); @@ -161,7 +175,8 @@ public void testTimeCat() throws Exception { ap( "tabby", "calico" - ) + ), + 2 ); // 4 docs no new collections @@ -183,7 +198,8 @@ public void testTimeCat() throws Exception { ap( "tabby", "calico" - ) + ), + 6 ); // 4 docs 2 new collections, in random order and maybe not using the alias @@ -206,7 +222,8 @@ public void testTimeCat() throws Exception { ap( "tabby", "calico" - ) + ), + 10 ); // now test with async pre-create. @@ -236,7 +253,8 @@ public void testTimeCat() throws Exception { "shorthair", "tabby", "calico" - ) + ), + 12 ); addDocsAndCommit(false, @@ -268,7 +286,8 @@ Here we need to be testing that something that should not be created (extra pree "shorthair", "tabby", "calico" - ) + ), + 14 ); // now test with auto-delete. @@ -298,7 +317,8 @@ Here we need to be testing that something that should not be created (extra pree "shorthair", "tabby", "calico" - ) + ), + 16 ); // have to only send to alias here since one of the collections will be deleted. @@ -326,25 +346,30 @@ Here we need to be testing that something that should not be created (extra pree "shorthair", "tabby", "calico" - ) + ), 18 ); + } + + private void checkDocsInRightCollections(int expected) throws SolrServerException, IOException { // verify that all the documents ended up in the right collections. QueryResponse resp = solrClient.query(getAlias(), params( "q", "*:*", "rows", "100", - "fl","*,[shard]", + "fl", "*,[shard]", "sort", "id asc" )); SolrDocumentList results = resp.getResults(); - assertEquals(18, results.getNumFound()); + assertEquals(expected, results.getNumFound()); for (SolrDocument result : results) { String shard = String.valueOf(result.getFieldValue("[shard]")); String cat = String.valueOf(result.getFieldValue("cat_s")); Date date = (Date) result.getFieldValue("timestamp_dt"); String day = date.toInstant().toString().split("T")[0]; - assertTrue(shard.contains(cat)); - assertTrue(shard.contains(day)); + assertTrue("Cat=" + cat + " should not be in Shard=" + shard + + " DocId=" + result.getFieldValue("id"), shard.contains(cat)); + assertTrue("Day=" + day + " should not be in Shard=" + shard + + " DocId=" + result.getFieldValue("id"), shard.contains(day)); } } @@ -383,7 +408,8 @@ public void testCatTime() throws Exception { ), ap( "tabby" - ) + ), + 1 ); addDocsAndCommit(true, newDoc("calico", "2019-07-02T00:00:00Z")); @@ -399,11 +425,12 @@ public void testCatTime() throws Exception { ap( "tabby", "calico" - ) + ), + 2 ); testFailedDocument("shorthair", "2017-10-23T00:00:00Z", "couldn't be routed" ); - testFailedDocument("shorthair", "2020-10-23T00:00:00Z", "too far in the future" ); + testFailedDocument("shorthair", "2090-10-23T00:00:00Z", "too far in the future" ); testFailedDocument(null, "2019-07-02T00:00:00Z", "Route value is null"); testFailedDocument("foo__CRA__bar", "2019-07-02T00:00:00Z", "7 character sequence __CRA__"); testFailedDocument("fóóCRAóóbar", "2019-07-02T00:00:00Z", "7 character sequence __CRA__"); @@ -419,7 +446,8 @@ public void testCatTime() throws Exception { ap( "tabby", "calico" - ) + ), + 2 ); // 4 docs no new collections @@ -441,7 +469,8 @@ public void testCatTime() throws Exception { ap( "tabby", "calico" - ) + ), + 6 ); // 4 docs 2 new collections, in random order and maybe not using the alias @@ -466,7 +495,8 @@ public void testCatTime() throws Exception { ap( "tabby", "calico" - ) + ), + 10 ); // now test with async pre-create. @@ -496,7 +526,8 @@ public void testCatTime() throws Exception { "shorthair", "tabby", "calico" - ) + ), + 12 ); addDocsAndCommit(false, @@ -528,7 +559,8 @@ Here we need to be testing that something that should not be created (extra pree "shorthair", "tabby", "calico" - ) + ), + 14 ); // now test with auto-delete. @@ -558,7 +590,8 @@ Here we need to be testing that something that should not be created (extra pree "shorthair", "tabby", "calico" - ) + ), + 16 ); addDocsAndCommit(true, @@ -585,27 +618,10 @@ Here we need to be testing that something that should not be created (extra pree "shorthair", "tabby", "calico" - ) + ), + 18 ); - // verify that all the documents ended up in the right collections. - QueryResponse resp = solrClient.query(getAlias(), params( - "q", "*:*", - "rows", "100", - "fl","*,[shard]", - "sort", "id asc" - )); - SolrDocumentList results = resp.getResults(); - assertEquals(18, results.getNumFound()); - for (SolrDocument result : results) { - String shard = String.valueOf(result.getFieldValue("[shard]")); - String cat = String.valueOf(result.getFieldValue("cat_s")); - Date date = (Date) result.getFieldValue("timestamp_dt"); - String day = date.toInstant().toString().split("T")[0]; - assertTrue(shard.contains(cat)); - assertTrue(shard.contains(day)); - } - } public String catTimeDraColFor(String category, String timestamp) { @@ -621,9 +637,11 @@ public String timeCatDraColFor(String timestamp, String category) { * * @param expectedCols the collections we expect to see * @param categories the categories added thus far + * @param numDocsExpected the expected number of documents across all shards */ - private void assertCatTimeInvariants(String[] expectedCols, String[] categories) throws Exception { - final int expectNumFound = lastDocId - numDocsDeletedOrFailed; //lastDocId is effectively # generated docs + private void assertCatTimeInvariants(String[] expectedCols, String[] categories, int numDocsExpected) throws Exception { + // verify that all the documents ended up in the right collections. + checkDocsInRightCollections(numDocsExpected); int totalNumFound = 0; final List cols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(getSaferTestName()); @@ -675,7 +693,11 @@ private void assertCatTimeInvariants(String[] expectedCols, String[] categories) } - assertEquals(expectNumFound, totalNumFound); + assertEquals(numDocsExpected, totalNumFound); + + // also sanity check that our concepts of deleted/failed docs and doc id's have remained consistent. + final int properNumFound = lastDocId - numDocsDeletedOrFailed; + assertEquals(properNumFound, totalNumFound); assertEquals("COLS FOUND:" + cols, expectedCols.length, cols.size()); } diff --git a/solr/core/src/test/org/apache/solr/update/processor/RoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/RoutedAliasUpdateProcessorTest.java index 906e393c66cd..5fd85fa60610 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/RoutedAliasUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/RoutedAliasUpdateProcessorTest.java @@ -28,7 +28,6 @@ import java.util.TreeSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.stream.Collectors; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; @@ -61,7 +60,7 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; -@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13696") +//@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13696") @Ignore // don't try too run abstract base class public abstract class RoutedAliasUpdateProcessorTest extends SolrCloudTestCase { @@ -265,8 +264,6 @@ void addDocsAndCommit(boolean aliasOnly, SolrInputDocument... solrInputDocuments collections.addAll(new CollectionAdminRequest.ListAliases().process(getSolrClient()).getAliasesAsLists().get(getAlias())); } - int commitWithin = random().nextBoolean() ? -1 : 500; // if -1, we commit explicitly instead - if (random().nextBoolean()) { // Send in separate threads. Choose random collection & solrClient ExecutorService exec = null; @@ -277,7 +274,7 @@ void addDocsAndCommit(boolean aliasOnly, SolrInputDocument... solrInputDocuments List> futures = new ArrayList<>(solrInputDocuments.length); for (SolrInputDocument solrInputDocument : solrInputDocuments) { String col = collections.get(random().nextInt(collections.size())); - futures.add(exec.submit(() -> solrClient.add(col, solrInputDocument, commitWithin))); + futures.add(exec.submit(() -> solrClient.add(col, solrInputDocument, -1))); } for (Future future : futures) { assertUpdateResponse(future.get()); @@ -294,34 +291,11 @@ void addDocsAndCommit(boolean aliasOnly, SolrInputDocument... solrInputDocuments // send in a batch. String col = collections.get(random().nextInt(collections.size())); try (CloudSolrClient solrClient = getCloudSolrClient(cluster)) { - assertUpdateResponse(solrClient.add(col, Arrays.asList(solrInputDocuments), commitWithin)); + assertUpdateResponse(solrClient.add(col, Arrays.asList(solrInputDocuments), -1)); } } String col = collections.get(random().nextInt(collections.size())); - if (commitWithin == -1) { - getSolrClient().commit(col); - } else { - // check that it all got committed eventually - String docsQ = - "{!terms f=id}" - + Arrays.stream(solrInputDocuments).map(d -> d.getFieldValue("id").toString()) - .collect(Collectors.joining(",")); - int numDocs = queryNumDocs(docsQ); - if (numDocs == solrInputDocuments.length) { - System.err.println("Docs committed sooner than expected. Bug or slow test env?"); - return; - } - // wait until it's committed - Thread.sleep(commitWithin); - for (int idx = 0; idx < 100; ++idx) { // Loop for up to 10 seconds waiting for commit to catch up - numDocs = queryNumDocs(docsQ); - if (numDocs == solrInputDocuments.length) break; - Thread.sleep(100); - } - - assertEquals("not committed. Bug or a slow test?", - solrInputDocuments.length, numDocs); - } + getSolrClient().commit(col); } void assertUpdateResponse(UpdateResponse rsp) { diff --git a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java index 51416013638a..5fa12956ea9c 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java @@ -59,7 +59,9 @@ import org.apache.zookeeper.Watcher; import org.apache.zookeeper.data.Stat; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,6 +74,8 @@ @LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13059") public class TimeRoutedAliasUpdateProcessorTest extends RoutedAliasUpdateProcessorTest { + private static final String origSysprop=System.getProperty("solr.disableConfigSetsCreateAuthChecks"); + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String alias = "myalias"; @@ -103,6 +107,20 @@ public void doAfter() throws Exception { shutdownCluster(); } + @BeforeClass + public static void setUpClass() { + System.setProperty("solr.disableConfigSetsCreateAuthChecks", "true"); + } + + @AfterClass + public static void cleanUpAfterClass() throws Exception { + if (origSysprop != null) { + System.setProperty("solr.disableConfigSetsCreateAuthChecks", origSysprop); + } else { + System.getProperties().remove("solr.disableConfigSetsCreateAuthChecks"); + } + } + @Slow @Test @LogLevel("org.apache.solr.update.processor.TimeRoutedAlias=DEBUG;org.apache.solr.cloud=DEBUG")