From 34528dbfff77905b7742b4822620b5634655acdd Mon Sep 17 00:00:00 2001 From: Mano Kovacs Date: Thu, 20 Sep 2018 16:54:18 +0200 Subject: [PATCH] SOLR-8335 HdfsLockFactory should eventually release index after crash. --- .../lucene/store/LockLostException.java | 43 +++ .../apache/lucene/index/TestIndexSorting.java | 2 +- lucene/ivy-versions.properties | 3 +- lucene/licenses/hamcrest-all-1.3.jar.sha1 | 1 + lucene/licenses/hamcrest-all-LICENSE-BSD.txt | 27 ++ lucene/licenses/hamcrest-all-NOTICE.txt | 2 + lucene/licenses/junit-4.10.jar.sha1 | 1 - lucene/licenses/junit-4.11.jar.sha1 | 1 + .../lucene/index/memory/TestMemoryIndex.java | 2 +- lucene/test-framework/ivy.xml | 1 + .../model/TestMultipleAdditiveTreesModel.java | 4 +- .../java/org/apache/solr/core/SolrCore.java | 1 + .../solr/core/StandardDirectoryFactory.java | 3 + .../solr/store/hdfs/HdfsLockFactory.java | 356 ++++++++++++++++-- .../solr/cloud/RemoteQueryErrorTest.java | 2 +- .../apache/solr/cloud/hdfs/HdfsTestUtil.java | 52 ++- .../org/apache/solr/core/TestConfigSets.java | 2 +- .../apache/solr/core/TestCoreDiscovery.java | 2 +- .../org/apache/solr/core/TestSolrXml.java | 2 +- .../solr/store/hdfs/HdfsLockFactoryTest.java | 346 ++++++++++++++++- .../org/apache/solr/update/PeerSyncTest.java | 2 +- .../update/TestInPlaceUpdatesStandalone.java | 3 +- .../org/apache/solr/update/UpdateLogTest.java | 2 +- .../SSLCredentialProviderFactoryTest.java | 22 +- solr/licenses/junit-4.10.jar.sha1 | 1 - solr/licenses/junit-4.11.jar.sha1 | 1 + .../solr/client/solrj/SolrExampleTests.java | 2 +- .../apache/solr/SolrIgnoredThreadsFilter.java | 6 + .../java/org/apache/solr/SolrTestCaseJ4.java | 18 +- 29 files changed, 818 insertions(+), 92 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/store/LockLostException.java create mode 100644 lucene/licenses/hamcrest-all-1.3.jar.sha1 create mode 100644 lucene/licenses/hamcrest-all-LICENSE-BSD.txt create mode 100644 lucene/licenses/hamcrest-all-NOTICE.txt delete mode 100644 lucene/licenses/junit-4.10.jar.sha1 create mode 100644 lucene/licenses/junit-4.11.jar.sha1 delete mode 100644 solr/licenses/junit-4.10.jar.sha1 create mode 100644 solr/licenses/junit-4.11.jar.sha1 diff --git a/lucene/core/src/java/org/apache/lucene/store/LockLostException.java b/lucene/core/src/java/org/apache/lucene/store/LockLostException.java new file mode 100644 index 000000000000..5a4f2b44bf38 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/store/LockLostException.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.IOException; + +/** + * This exception is thrown when the write.lock + * is lost. This + * happens when a writer is unable to keep the lock updated + * (cannot hold the lock) and another writer could successfully + * open the index. In such cases it is realized that the lock + * is not already owned and such an exception is thrown. + * @see LockFactory#obtainLock(Directory, String) + */ +public class LockLostException extends IOException { + public LockLostException(String message) { + super(message); + } + + public LockLostException(String message, Throwable cause) { + super(message, cause); + + } + + public LockLostException(Throwable cause) { + super(cause); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java index 3857a97d24c9..b5a6ff9b5283 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java @@ -86,7 +86,7 @@ import org.apache.lucene.util.TestUtil; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; -import static org.junit.internal.matchers.StringContains.containsString; +import static org.hamcrest.Matchers.containsString; public class TestIndexSorting extends LuceneTestCase { static class AssertingNeedsIndexSortCodec extends FilterCodec { diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index f53544e12b9b..a31db1e3f418 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -77,7 +77,7 @@ io.prometheus.version = 0.2.0 /javax.activation/activation = 1.1.1 /javax.servlet/javax.servlet-api = 3.1.0 -/junit/junit = 4.10 +/junit/junit = 4.11 /mecab/mecab-ipadic = 2.7.0-20070801 /mecab/mecab-ko-dic = 2.0.3-20170922 @@ -259,6 +259,7 @@ org.eclipse.jetty.version = 9.4.11.v20180605 org.gagravarr.vorbis.java.version = 0.8 /org.gagravarr/vorbis-java-core = ${org.gagravarr.vorbis.java.version} /org.gagravarr/vorbis-java-tika = ${org.gagravarr.vorbis.java.version} +/org.hamcrest/hamcrest-all = 1.3 /org.hsqldb/hsqldb = 2.4.0 /org.jdom/jdom2 = 2.0.6 diff --git a/lucene/licenses/hamcrest-all-1.3.jar.sha1 b/lucene/licenses/hamcrest-all-1.3.jar.sha1 new file mode 100644 index 000000000000..125f175a9c8e --- /dev/null +++ b/lucene/licenses/hamcrest-all-1.3.jar.sha1 @@ -0,0 +1 @@ +63a21ebc981131004ad02e0434e799fd7f3a8d5a diff --git a/lucene/licenses/hamcrest-all-LICENSE-BSD.txt b/lucene/licenses/hamcrest-all-LICENSE-BSD.txt new file mode 100644 index 000000000000..dcdcc423479c --- /dev/null +++ b/lucene/licenses/hamcrest-all-LICENSE-BSD.txt @@ -0,0 +1,27 @@ +BSD License + +Copyright (c) 2000-2006, www.hamcrest.org +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. Redistributions in binary form must reproduce +the above copyright notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the distribution. + +Neither the name of Hamcrest nor the names of its contributors may be used to endorse +or promote products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. diff --git a/lucene/licenses/hamcrest-all-NOTICE.txt b/lucene/licenses/hamcrest-all-NOTICE.txt new file mode 100644 index 000000000000..79871d009228 --- /dev/null +++ b/lucene/licenses/hamcrest-all-NOTICE.txt @@ -0,0 +1,2 @@ +Java Hamcrest is licensed under BSD License. +See http://hamcrest.org/ \ No newline at end of file diff --git a/lucene/licenses/junit-4.10.jar.sha1 b/lucene/licenses/junit-4.10.jar.sha1 deleted file mode 100644 index 875e26cee229..000000000000 --- a/lucene/licenses/junit-4.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e4f1766ce7404a08f45d859fb9c226fc9e41a861 diff --git a/lucene/licenses/junit-4.11.jar.sha1 b/lucene/licenses/junit-4.11.jar.sha1 new file mode 100644 index 000000000000..409d4c92e85d --- /dev/null +++ b/lucene/licenses/junit-4.11.jar.sha1 @@ -0,0 +1 @@ +4e031bb61df09069aeb2bffb4019e7a5034a4ee0 diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java index 94cf97448f66..ea84e2ca8a1c 100644 --- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java +++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java @@ -76,7 +76,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; -import static org.junit.internal.matchers.StringContains.containsString; +import static org.hamcrest.Matchers.containsString; public class TestMemoryIndex extends LuceneTestCase { diff --git a/lucene/test-framework/ivy.xml b/lucene/test-framework/ivy.xml index a51716c86986..859aa9bff25d 100644 --- a/lucene/test-framework/ivy.xml +++ b/lucene/test-framework/ivy.xml @@ -24,6 +24,7 @@ + diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestMultipleAdditiveTreesModel.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestMultipleAdditiveTreesModel.java index edc0e24e2510..b0dec200eeda 100644 --- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestMultipleAdditiveTreesModel.java +++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/model/TestMultipleAdditiveTreesModel.java @@ -16,14 +16,14 @@ */ package org.apache.solr.ltr.model; -import static org.junit.internal.matchers.StringContains.containsString; - import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.ltr.TestRerankBase; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import static org.hamcrest.Matchers.containsString; + public class TestMultipleAdditiveTreesModel extends TestRerankBase { diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 6e1303927459..0584e51b6946 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -1011,6 +1011,7 @@ public SolrCore(CoreContainer coreContainer, String name, String dataDir, SolrCo log.error("Error while closing", t); } + log.error("Error creating core", e); throw new SolrException(ErrorCode.SERVER_ERROR, e.getMessage(), e); } finally { // allow firstSearcher events to fire and make sure it is released diff --git a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java index 1bc4914cfcf8..2b34ba9581bd 100644 --- a/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java +++ b/solr/core/src/java/org/apache/solr/core/StandardDirectoryFactory.java @@ -35,6 +35,7 @@ import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.solr.common.SolrException; +import org.apache.solr.store.hdfs.HdfsLockFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,6 +73,8 @@ protected LockFactory createLockFactory(String rawLockType) throws IOException { return new SingleInstanceLockFactory(); case DirectoryFactory.LOCK_TYPE_NONE: return NoLockFactory.INSTANCE; + case DirectoryFactory.LOCK_TYPE_HDFS: + return HdfsLockFactory.INSTANCE; default: throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unrecognized lockType: " + rawLockType); diff --git a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java index c62b5f4a4aae..576f5d36f235 100644 --- a/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java +++ b/solr/core/src/java/org/apache/solr/store/hdfs/HdfsLockFactory.java @@ -18,29 +18,118 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.LockLostException; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.LockReleaseFailedException; import org.apache.solr.common.util.IOUtils; +import org.apache.solr.util.DefaultSolrThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class HdfsLockFactory extends LockFactory { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - + public static final long DEFAULT_LOCK_ACQUIRE_TIMEOUT = 90_000L; + public static final long DEFAULT_LOCK_HOLD_TIMEOUT = DEFAULT_LOCK_ACQUIRE_TIMEOUT / 2; + public static final long DEFAULT_UPDATE_DELAY = 5000L; public static final HdfsLockFactory INSTANCE = new HdfsLockFactory(); - - private HdfsLockFactory() {} - + public static final String HDFS_LOCK_UPDATE_PREFIX = "hdfs-lock-update"; + public static final int HDFS_LOCK_UPDATE_THREADS = 5; + + public static final String LOCK_HOLD_TIMEOUT_KEY = "hdfs.lock.update.hold.timeout"; + public static final String UPDATE_DELAY_KEY = "hdfs.lock.update.delay"; + public static final String HDFS_LOCK_ACQUIRE_TIMEOUT_KEY = "hdfs.lock.acquire.timeout"; + private static final int MAX_META_LOCK_DEPTH = 10; + + private SleepService sleeper; + + private volatile ScheduledExecutorService executor; + private Consumer exceptionHandler; + private final Object executorGuard = new Object(); + + private HdfsLockFactory() { + reset(); + } + + void reset() { + sleeper = m -> { + try { + Thread.sleep(m); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }; + executor = null; + } + + public void stopBackgroundTasks() { + if(executor == null) + return; + try { + ScheduledExecutorService executorToShutDown; + synchronized (executorGuard) { + executorToShutDown = executor; + executor = null; + } + executorToShutDown.shutdownNow(); + executorToShutDown.awaitTermination(3, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + private ScheduledExecutorService newExecutor() { + return Executors.newScheduledThreadPool(HDFS_LOCK_UPDATE_THREADS, new DefaultSolrThreadFactory(HDFS_LOCK_UPDATE_PREFIX)); + } + + private ScheduledExecutorService getExecutor() { + if(executor != null) + return executor; + synchronized (executorGuard) { + if (executor == null) + executor = newExecutor(); + return executor; + } + } + + + public void setExecutorService(ScheduledExecutorService executor) { + this.executor = executor; + } + + public void setSleeper(SleepService sleeper) { + this.sleeper = sleeper; + } + + public void setExceptionHandler(Consumer exceptionHandler) { + this.exceptionHandler = exceptionHandler; + } + + + public interface SleepService { + public void sleep(long millis); + } + @Override public Lock obtainLock(Directory dir, String lockName) throws IOException { if (!(dir instanceof HdfsDirectory)) { @@ -50,64 +139,132 @@ public Lock obtainLock(Directory dir, String lockName) throws IOException { final Configuration conf = hdfsDir.getConfiguration(); final Path lockPath = hdfsDir.getHdfsDirPath(); final Path lockFile = new Path(lockPath, lockName); - - FSDataOutputStream file = null; + final FileSystem fs = FileSystem.get(lockPath.toUri(), conf); + log.info("Trying to lock on " + fs.getClass().getName() + " the following path:" + lockPath); + HdfsLock hdfsLock; + createDirs(lockPath, fs); while (true) { try { - if (!fs.exists(lockPath)) { - boolean success = fs.mkdirs(lockPath); - if (!success) { - throw new RuntimeException("Could not create directory: " + lockPath); - } - } else { - // just to check for safe mode - fs.mkdirs(lockPath); - } - - file = fs.create(lockFile, false); + log.info("Trying to create directory {}, exists: {}", lockPath.getName(), fs.exists(lockFile)); + hdfsLock = new HdfsLock(conf, lockFile, fs); + hdfsLock.init(); break; - } catch (FileAlreadyExistsException e) { - throw new LockObtainFailedException("Cannot obtain lock file: " + lockFile, e); - } catch (RemoteException e) { - if (e.getClassName().equals( - "org.apache.hadoop.hdfs.server.namenode.SafeModeException")) { - log.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again."); - try { - Thread.sleep(5000); - } catch (InterruptedException e1) { - Thread.interrupted(); - } - continue; + } catch (SafeModeException e) { + log.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again."); + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + Thread.interrupted(); } - throw new LockObtainFailedException("Cannot obtain lock file: " + lockFile, e); } catch (IOException e) { throw new LockObtainFailedException("Cannot obtain lock file: " + lockFile, e); - } finally { - IOUtils.closeQuietly(file); } } + hdfsLock.startScheduledUpdate(); + return hdfsLock; + } + + private int getLockDepth(Path lockFile, FileSystem fs) throws IOException { + Path l = lockFile; + int i; + for (i = 0; i < MAX_META_LOCK_DEPTH;i++) { + if (!fs.exists(l)) break; + l = getMetaLock(l); + } + return i; + } + + private void createDirs(Path lockPath, FileSystem fs) throws IOException { + if (!fs.exists(lockPath)) { + boolean success = fs.mkdirs(lockPath); + if (!success) { + throw new RuntimeException("Could not create directory: " + lockPath); + } + } else { + // just to check for safe mode + fs.mkdirs(lockPath); + } + } + + private Path getMetaLock(Path lockFile, int lockDepth) { + Path monitor = lockFile; + for(int i= 1; i scheduledFileUpdate; + private long cycle = 0; + long lastLockUpdate = 0; + private AtomicReference> deleteGuard = new AtomicReference(Optional.empty()); + private boolean lockExpired = false; + + HdfsLock(Configuration conf, Path lockFile, FileSystem fs) { this.conf = conf; this.lockFile = lockFile; + this.fs = fs; + updateDelay = getUpdateDelay(); + updateTimeoutHold = Long.valueOf(System.getProperty(LOCK_HOLD_TIMEOUT_KEY, String.valueOf(DEFAULT_LOCK_HOLD_TIMEOUT))); } @Override public void close() throws IOException { + final FileSystem fs = FileSystem.get(lockFile.toUri(), conf); if (closed) { return; } - final FileSystem fs = FileSystem.get(lockFile.toUri(), conf); + closed = true; + stopUpdates(); + deleteGuard(); try { if (fs.exists(lockFile) && !fs.delete(lockFile, false)) { throw new LockReleaseFailedException("failed to delete: " + lockFile); @@ -119,12 +276,129 @@ public void close() throws IOException { @Override public void ensureValid() throws IOException { - // no idea how to implement this on HDFS + UUID actualId = readLockInfo(lockFile, fs).id; + if ( !actualId.equals(id) ) + throw new LockLostException("Another process took ownership of the lock. Lock id: " + actualId + " own id: " + id); } @Override public String toString() { return "HdfsLock(lockFile=" + lockFile + ")"; } + + void stopUpdates() { + scheduledFileUpdate.cancel(true); + } + + void startScheduledUpdate() { + scheduledFileUpdate = getExecutor().scheduleWithFixedDelay(() -> { + try { + if(lockExpired) + reAcquireLock(); + else { + ensureValid(); + createLockFile(true); + } + } catch (IOException | RuntimeException e) { + if (System.nanoTime() - lastLockUpdate >= updateTimeoutHold * 1_000_000) { + lockExpired = true; + } + exceptionHandler.accept(e); + throw new RuntimeException(e); + } + }, updateDelay, updateDelay, TimeUnit.MILLISECONDS); + } + + private void reAcquireLock() { + try { + int i = getLockDepth(lockFile, fs); + if(i > 0) { + long lockTimeout = Long.valueOf(System.getProperty(HDFS_LOCK_ACQUIRE_TIMEOUT_KEY, String.valueOf(DEFAULT_LOCK_ACQUIRE_TIMEOUT))); + Path monitor = getMetaLock(lockFile, i); + LockInfo ownLockInfo = new LockInfo(id, cycle - 1); + LockInfo actual = expectUnchangedLockInfo(monitor, ownLockInfo, lockTimeout); + if(ownLockInfo.equals(actual)) { + removeStaleLock(lockFile, i); + createLockFile(false); + lockExpired = false; + } else { + exceptionHandler.accept(new LockLostException("Another process took ownership of the lock. Lock id: " + actual.id + " own id: " + id)); + stopUpdates(); + } + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void init() throws IOException { + int i = getLockDepth(lockFile, fs); + if(i > 0) { + removeIfUntouched(lockFile, i); + } + createLockFile(false); + + } + + private void removeIfUntouched(Path lockFile, int lockDepth) throws IOException { + long lockTimeout = Long.valueOf(System.getProperty(HDFS_LOCK_ACQUIRE_TIMEOUT_KEY, String.valueOf(DEFAULT_LOCK_ACQUIRE_TIMEOUT))); + Path monitor = getMetaLock(lockFile, lockDepth); + boolean expired = waitUntilExpires(lockTimeout, monitor); + if(expired) { + removeStaleLock(lockFile, lockDepth); + } + } + + private void removeStaleLock(Path lockFile, int lockDepth) throws IOException { + deleteGuard.set(Optional.of(getMetaLock(lockFile, lockDepth + 1))); + writeLockFile(deleteGuard.get().get(), false); + Path toDelete = lockFile; + for(int i= 0; i guard = deleteGuard.getAndSet(Optional.empty()); + if(guard.isPresent()) + return fs.delete(guard.get(), false); + return false; + } + + + private void createLockFile(boolean overwrite) throws IOException { + writeLockFile(lockFile, overwrite); + lastLockUpdate = System.nanoTime(); + } + + private void writeLockFile(Path lockFile, boolean overwrite) throws IOException { + try (FSDataOutputStream file = fs.create(lockFile, overwrite)) { + file.writeUTF(id.toString()); + file.writeLong(cycle++); + } + } + } + + private long getUpdateDelay() { + return Long.valueOf(System.getProperty(UPDATE_DELAY_KEY, String.valueOf(DEFAULT_UPDATE_DELAY))); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/RemoteQueryErrorTest.java b/solr/core/src/test/org/apache/solr/cloud/RemoteQueryErrorTest.java index 54503bfe1ea3..4965478eface 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RemoteQueryErrorTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RemoteQueryErrorTest.java @@ -24,7 +24,7 @@ import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.internal.matchers.StringContains.containsString; +import static org.hamcrest.Matchers.containsString; /** * Verify that remote (proxied) queries return proper error messages diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java index 703832822033..d966e30c0b48 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java @@ -82,18 +82,8 @@ public static MiniDFSCluster setupClass(String dir, boolean safeModeTesting, boo ThreadLocalRandom.setInitialSeedUniquifier(1L); int dataNodes = Integer.getInteger("tests.hdfs.numdatanodes", 2); - - Configuration conf = new Configuration(); - conf.set("dfs.block.access.token.enable", "false"); - conf.set("dfs.permissions.enabled", "false"); - conf.set("hadoop.security.authentication", "simple"); - conf.set("hdfs.minidfs.basedir", dir + File.separator + "hdfsBaseDir"); - conf.set("dfs.namenode.name.dir", dir + File.separator + "nameNodeNameDir"); - conf.setBoolean("fs.hdfs.impl.disable.cache", true); - - System.setProperty("test.build.data", dir + File.separator + "hdfs" + File.separator + "build"); - System.setProperty("test.cache.data", dir + File.separator + "hdfs" + File.separator + "cache"); - System.setProperty("solr.lock.type", DirectoryFactory.LOCK_TYPE_HDFS); + + Configuration conf = getDefaultConfiguration(dir); System.setProperty("solr.hdfs.blockcache.global", Boolean.toString(LuceneTestCase.random().nextBoolean())); @@ -169,6 +159,44 @@ public void run() { return dfsCluster; } + public static MiniDFSCluster setupClassBasic(String dir) throws Exception { + LuceneTestCase.assumeFalse("HDFS tests were disabled by -Dtests.disableHdfs", + Boolean.parseBoolean(System.getProperty("tests.disableHdfs", "false"))); + + savedLocale = Locale.getDefault(); + // TODO: we HACK around HADOOP-9643 + Locale.setDefault(Locale.ENGLISH); + + Configuration conf = getDefaultConfiguration(dir); + + + System.setProperty("solr.hdfs.blockcache.global", "true"); + + MiniDFSCluster dfsCluster = new MiniDFSCluster(conf, 1, true, null); + + System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home")); + dfsCluster.waitActive(); + + SolrTestCaseJ4.useFactory("org.apache.solr.core.HdfsDirectoryFactory"); + + return dfsCluster; + } + + private static Configuration getDefaultConfiguration(String dir) { + Configuration conf = new Configuration(); + conf.set("dfs.block.access.token.enable", "false"); + conf.set("dfs.permissions.enabled", "false"); + conf.set("hadoop.security.authentication", "simple"); + conf.set("hdfs.minidfs.basedir", dir + File.separator + "hdfsBaseDir"); + conf.set("dfs.namenode.name.dir", dir + File.separator + "nameNodeNameDir"); + conf.setBoolean("fs.hdfs.impl.disable.cache", true); + + System.setProperty("test.build.data", dir + File.separator + "hdfs" + File.separator + "build"); + System.setProperty("test.cache.data", dir + File.separator + "hdfs" + File.separator + "cache"); + System.setProperty("solr.lock.type", DirectoryFactory.LOCK_TYPE_HDFS); + return conf; + } + public static Configuration getClientConfiguration(MiniDFSCluster dfsCluster) { if (dfsCluster.getNameNodeInfos().length > 1) { Configuration conf = new Configuration(); diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigSets.java b/solr/core/src/test/org/apache/solr/core/TestConfigSets.java index ae11fd67466a..905c53098b1d 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfigSets.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfigSets.java @@ -33,7 +33,7 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.core.Is.is; -import static org.junit.internal.matchers.StringContains.containsString; +import static org.hamcrest.Matchers.containsString; public class TestConfigSets extends SolrTestCaseJ4 { diff --git a/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java b/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java index 4e944c3c10a6..2e45db0f1113 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java +++ b/solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java @@ -43,7 +43,7 @@ import static org.apache.solr.core.CoreContainer.INITIAL_CORE_LOAD_COMPLETE; import static org.apache.solr.core.CoreContainer.LOAD_COMPLETE; import static org.hamcrest.CoreMatchers.not; -import static org.junit.internal.matchers.StringContains.containsString; +import static org.hamcrest.Matchers.containsString; public class TestCoreDiscovery extends SolrTestCaseJ4 { diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrXml.java b/solr/core/src/test/org/apache/solr/core/TestSolrXml.java index 031b4c8d0e14..37ba97e190f9 100644 --- a/solr/core/src/test/org/apache/solr/core/TestSolrXml.java +++ b/solr/core/src/test/org/apache/solr/core/TestSolrXml.java @@ -34,7 +34,7 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; -import static org.junit.internal.matchers.StringContains.containsString; +import static org.hamcrest.Matchers.containsString; public class TestSolrXml extends SolrTestCaseJ4 { diff --git a/solr/core/src/test/org/apache/solr/store/hdfs/HdfsLockFactoryTest.java b/solr/core/src/test/org/apache/solr/store/hdfs/HdfsLockFactoryTest.java index 452c1f406cf7..2487a29fe4df 100644 --- a/solr/core/src/test/org/apache/solr/store/hdfs/HdfsLockFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/store/hdfs/HdfsLockFactoryTest.java @@ -17,46 +17,175 @@ package org.apache.solr.store.hdfs; import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.net.ConnectException; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Delayed; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.lucene.store.Lock; +import org.apache.lucene.store.LockLostException; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.TestRuleRestoreSystemProperties; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.cloud.hdfs.HdfsTestUtil; import org.apache.solr.util.BadHdfsThreadsFilter; +import org.hamcrest.Matchers; +import org.junit.After; import org.junit.AfterClass; +import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.mockito.internal.util.reflection.FieldSetter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.Arrays.asList; +import static org.apache.solr.store.hdfs.HdfsLockFactory.DEFAULT_LOCK_HOLD_TIMEOUT; +import static org.apache.solr.store.hdfs.HdfsLockFactory.DEFAULT_UPDATE_DELAY; +import static org.apache.solr.store.hdfs.HdfsLockFactory.LOCK_HOLD_TIMEOUT_KEY; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.BDDMockito.given; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.mock; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; @ThreadLeakFilters(defaultFilters = true, filters = { BadHdfsThreadsFilter.class // hdfs currently leaks thread(s) }) public class HdfsLockFactoryTest extends SolrTestCaseJ4 { - + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static MiniDFSCluster dfsCluster; + private static NameNode nameNode; + private static Configuration nameNodeConf; + private long waitTime = 0; + private List allTasks = new LinkedList<>(); + private HdfsDirectory dir; + private Path lockPath; + private HdfsLockFactory.HdfsLock lock = null; + private HdfsLockFactory.HdfsLock lock2 = null; + private Exception latestException = null; + private List allLocks = new LinkedList<>(); + private static MiniDFSCluster.DataNodeProperties dataNodeProperties; + + @Rule + public TestRuleRestoreSystemProperties p = new TestRuleRestoreSystemProperties(LOCK_HOLD_TIMEOUT_KEY); + private LockLostException lockLostException; + private List executing = new LinkedList<>(); @BeforeClass public static void beforeClass() throws Exception { - dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath()); + SolrTestCaseJ4.assumeWorkingMockito(); + dfsCluster = HdfsTestUtil.setupClassBasic(createTempDir().toFile().getAbsolutePath()); + nameNode = dfsCluster.getNameNode(); + nameNodeConf = dfsCluster.getConfiguration(0); } @AfterClass public static void afterClass() throws Exception { HdfsTestUtil.teardownClass(dfsCluster); dfsCluster = null; + nameNode = null; + dataNodeProperties = null; + } + + @Before + public void setupHdfsLockFactory() { + System.setProperty(LOCK_HOLD_TIMEOUT_KEY, String.valueOf(DEFAULT_LOCK_HOLD_TIMEOUT)); + Consumer exceptionHandler = e -> { + latestException = e; + if (e instanceof LockLostException) + lockLostException = (LockLostException) e; + }; + HdfsLockFactory.INSTANCE.setExceptionHandler(exceptionHandler); + ScheduledExecutorService exec = mock(ScheduledExecutorService.class); + given(exec.scheduleWithFixedDelay(any(Runnable.class), anyLong(), anyLong(), any(TimeUnit.class))) + .willAnswer(invocation -> { + Object[] a = invocation.getArguments(); + ScheduledTaskStub task = new ScheduledTaskStub((Runnable) a[0], (Long) a[1], (Long) a[2], (TimeUnit) a[3]); + allTasks.add(task); + return task; + }); + HdfsLockFactory.INSTANCE.setExecutorService(exec); + } + + @AfterClass + public static void resetTimeDependencies() throws Exception { + HdfsLockFactory.INSTANCE.reset(); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + startStoppedDatanode(); + String uri = HdfsTestUtil.getURI(dfsCluster); + lockPath = new Path(uri, "/basedir/lock"); + Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); + dir = new HdfsDirectory(lockPath, conf); + HdfsLockFactory.INSTANCE.setSleeper(m -> { + waitTime += m; + executeScheduledTasks(); + }); + } + + private void startStoppedDatanode() throws IOException, InterruptedException { + if (dataNodeProperties != null) { + dfsCluster.restartDataNode(dataNodeProperties, true); + for (int i = 0; i < 1200; i++) { + if (dfsCluster.getDataNodes().get(0).isDatanodeFullyStarted()) break; + Thread.sleep(100); + } + assertTrue(dfsCluster.getDataNodes().get(0).isDatanodeFullyStarted()); + dataNodeProperties = null; + } } - + + @After + public void releaseLocks() throws Exception { + allLocks.forEach(l -> { + try { + l.close(); + } catch (IOException ignored) { + } + }); + startStoppedDatanode(); + DistributedFileSystem fs = dfsCluster.getFileSystem(); + List lockFiles = asList(fs.listStatus(lockPath)); + lockFiles.forEach(s -> { + try { + fs.delete(s.getPath(), true); + } catch (IOException ignored) { + } + }); + } + @Test public void testBasic() throws IOException { String uri = HdfsTestUtil.getURI(dfsCluster); Path lockPath = new Path(uri, "/basedir/lock"); Configuration conf = HdfsTestUtil.getClientConfiguration(dfsCluster); HdfsDirectory dir = new HdfsDirectory(lockPath, conf); - + + try (Lock lock = dir.obtainLock("testlock")) { assert lock != null; try (Lock lock2 = dir.obtainLock("testlock")) { @@ -78,4 +207,211 @@ public void testBasic() throws IOException { } dir.close(); } + + @Test + public void testBasic_SecondLockFails() throws IOException { + lock = obtainLock(); + try { + lock2 = obtainLock(); + fail(); + } catch (LockObtainFailedException ignored) { + } + } + + @Test + public void testBasic_LockCanBeRepeated() throws IOException { + lock = obtainLock(); + failObtainingLock(); + lock.close(); + lock = obtainLock(); + failObtainingLock(); + } + + @Test + public void lockIsNotLostIfCannotUpdateForAWhile() throws Exception { + System.setProperty(LOCK_HOLD_TIMEOUT_KEY, "0"); + lock = obtainLock(); + disconnectFromHdfs(lock); + executeScheduledTasks(); + assertFalse(lockLostReported()); + } + + @Test + public void lockCanBeTakenOverIfNotRefreshed() throws IOException { + lock = obtainLock(); + disconnectFromHdfs(lock); + lock2 = obtainLock(); + + assertThat(waitTime, Matchers.greaterThanOrEqualTo(5000L)); + } + + @Test + public void lockFailsEarlyDueToPollingItsStatus() throws IOException { + lock = obtainLock(); + disconnectFromHdfs(lock); + HdfsLockFactory.INSTANCE.setSleeper(t -> { + this.waitTime += t; + if (waitTime >= 3 * DEFAULT_UPDATE_DELAY) + reconnectHdfs(lock); + executeScheduledTasks(); + }); + failObtainingLock(); + assertThat(waitTime, equalTo(DEFAULT_UPDATE_DELAY * 3)); + } + + @Test + public void lockIsLostIfIdIsOverWrittenInIt() throws IOException { + lock = obtainLock(); + disconnectFromHdfs(lock); + lock2 = obtainLock(); + reconnectHdfs(lock); + executeScheduledTasks(); + assertTrue(lockLostReported()); + } + + @Test + public void willNotReportLockLostIfDisconnected() throws Exception { + lock = obtainLock(); + for (int i = 0; i < 20; i++) { + executeScheduledTasks(); + } + assertFalse(lockLostReported()); + } + + @Test + public void only1outOf2getsTheLock() throws Exception { + lock = obtainLock(); + disconnectFromHdfs(lock); + CountDownLatch countDown = new CountDownLatch(2); + final AtomicInteger successful = new AtomicInteger(0); + new Thread(() -> { + try { + obtainLock(); + successful.incrementAndGet(); + } catch (Exception ignored) { + + } finally { + countDown.countDown(); + } + }).run(); + new Thread(() -> { + try { + obtainLock(); + successful.incrementAndGet(); + } catch (Exception ignored) { + + } finally { + countDown.countDown(); + } + }).run(); + countDown.await(10_000, TimeUnit.MILLISECONDS); + assertThat(successful.get(), equalTo(1)); + } + + private boolean lockLostReported() { + return lockLostException != null; + } + + private void failObtainingLock() throws IOException { + try { + obtainLock(); + fail(); + } catch (LockObtainFailedException ignored) { + } + } + + private void reconnectHdfs(HdfsLockFactory.HdfsLock lock) { + try { + setFileSystem(lock, dfsCluster.getFileSystem()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private HdfsLockFactory.HdfsLock obtainLock() throws IOException { + HdfsLockFactory.HdfsLock lock = (HdfsLockFactory.HdfsLock) dir.obtainLock("testlock"); + allLocks.add(lock); + return lock; + } + + private void disconnectFromHdfs(HdfsLockFactory.HdfsLock lock) { + DistributedFileSystem failing = mock(DistributedFileSystem.class, i -> { + throw new ConnectException("Injected failure"); + }); + setFileSystem(lock, failing); + } + + private void setFileSystem(HdfsLockFactory.HdfsLock lock, DistributedFileSystem failing) { + try { + FieldSetter.setField(lock, HdfsLockFactory.HdfsLock.class.getDeclaredField("fs"), failing); + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + + private class ScheduledTaskStub implements ScheduledFuture { + Runnable task; + long inititalDelay; + long delay; + TimeUnit unit; + private boolean done = false; + + private ScheduledTaskStub(Runnable task, long inititalDelay, long delay, TimeUnit unit) { + this.task = task; + this.inititalDelay = inititalDelay; + this.delay = delay; + this.unit = unit; + } + + @Override + public long getDelay(TimeUnit unit) { + return delay; + } + + @Override + public int compareTo(Delayed o) { + return 0; + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return allTasks.remove(ScheduledTaskStub.this); + } + + @Override + public boolean isCancelled() { + return allTasks.contains(ScheduledTaskStub.this); + } + + @Override + public boolean isDone() { + return done; + } + + @Override + public T get() throws InterruptedException, ExecutionException { + return null; + } + + @Override + public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + return null; + } + + } + + private void executeScheduledTasks() { + LinkedList waitList = new LinkedList<>(allTasks); + waitList.removeAll(executing); + waitList.forEach(sch -> { + try { + executing.add(sch); + sch.task.run(); + } catch (RuntimeException ignored) { + } finally { + executing.remove(sch); + } + }); + } + } diff --git a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java index 848d1bc975fa..dd054141833c 100644 --- a/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java +++ b/solr/core/src/test/org/apache/solr/update/PeerSyncTest.java @@ -38,7 +38,7 @@ import org.apache.solr.update.processor.DistributedUpdateProcessor; import org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase; import org.junit.Test; -import static org.junit.internal.matchers.StringContains.containsString; +import static org.hamcrest.Matchers.containsString; @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") public class PeerSyncTest extends BaseDistributedSearchTestCase { diff --git a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java index 5986870f9085..b8d5a7ec879b 100644 --- a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java +++ b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesStandalone.java @@ -18,6 +18,7 @@ package org.apache.solr.update; + import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -53,7 +54,7 @@ import org.junit.Test; import static org.apache.solr.update.UpdateLogTest.buildAddUpdateCommand; -import static org.junit.internal.matchers.StringContains.containsString; +import static org.hamcrest.Matchers.containsString; /** diff --git a/solr/core/src/test/org/apache/solr/update/UpdateLogTest.java b/solr/core/src/test/org/apache/solr/update/UpdateLogTest.java index d2a57507310d..a8db9a632949 100644 --- a/solr/core/src/test/org/apache/solr/update/UpdateLogTest.java +++ b/solr/core/src/test/org/apache/solr/update/UpdateLogTest.java @@ -31,7 +31,7 @@ import org.junit.Test; import static org.apache.solr.common.params.CommonParams.VERSION_FIELD; -import static org.junit.internal.matchers.StringContains.containsString; +import static org.hamcrest.Matchers.containsString; public class UpdateLogTest extends SolrTestCaseJ4 { diff --git a/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java b/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java index 21e7f8a594d4..35e12cc73a48 100644 --- a/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/util/configuration/SSLCredentialProviderFactoryTest.java @@ -26,7 +26,7 @@ import org.junit.Test; import org.junit.rules.TestRule; -import static org.hamcrest.core.Is.is; +import static org.hamcrest.CoreMatchers.instanceOf; import static org.junit.Assert.assertThat; /** @@ -42,22 +42,22 @@ public class SSLCredentialProviderFactoryTest { public void testGetProvidersOrder() { SSLCredentialProviderFactory sut = getSut("sysprop;env"); List providers = sut.getProviders(); - assertThat(providers.get(0), is(SysPropSSLCredentialProvider.class)); - assertThat(providers.get(1), is(EnvSSLCredentialProvider.class)); + assertThat(providers.get(0), instanceOf(SysPropSSLCredentialProvider.class)); + assertThat(providers.get(1), instanceOf(EnvSSLCredentialProvider.class)); sut = getSut("env;sysprop"); providers = sut.getProviders(); - assertThat(providers.get(0), is(EnvSSLCredentialProvider.class)); - assertThat(providers.get(1), is(SysPropSSLCredentialProvider.class)); + assertThat(providers.get(0), instanceOf(EnvSSLCredentialProvider.class)); + assertThat(providers.get(1), instanceOf(SysPropSSLCredentialProvider.class)); } @Test public void testGetProvidersWithCustomProvider() { SSLCredentialProviderFactory sut = getSut("sysprop;class://" + CustomSSLCredentialProvider.class.getName() + ";env"); List providers = sut.getProviders(); - assertThat(providers.get(0), is(SysPropSSLCredentialProvider.class)); - assertThat(providers.get(1), is(CustomSSLCredentialProvider.class)); - assertThat(providers.get(2), is(EnvSSLCredentialProvider.class)); + assertThat(providers.get(0), instanceOf(SysPropSSLCredentialProvider.class)); + assertThat(providers.get(1), instanceOf(CustomSSLCredentialProvider.class)); + assertThat(providers.get(2), instanceOf(EnvSSLCredentialProvider.class)); } @Test(expected = RuntimeException.class) @@ -71,9 +71,9 @@ public void testGetProvidersBySysprop() { System.setProperty(SSLCredentialProviderFactory.PROVIDER_CHAIN_KEY, chain); SSLCredentialProviderFactory sut = new SSLCredentialProviderFactory(); List providers = sut.getProviders(); - assertThat(providers.get(0), is(SysPropSSLCredentialProvider.class)); - assertThat(providers.get(1), is(CustomSSLCredentialProvider.class)); - assertThat(providers.get(2), is(EnvSSLCredentialProvider.class)); + assertThat(providers.get(0), instanceOf(SysPropSSLCredentialProvider.class)); + assertThat(providers.get(1), instanceOf(CustomSSLCredentialProvider.class)); + assertThat(providers.get(2), instanceOf(EnvSSLCredentialProvider.class)); } private SSLCredentialProviderFactory getSut(String providerChain) { diff --git a/solr/licenses/junit-4.10.jar.sha1 b/solr/licenses/junit-4.10.jar.sha1 deleted file mode 100644 index 875e26cee229..000000000000 --- a/solr/licenses/junit-4.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e4f1766ce7404a08f45d859fb9c226fc9e41a861 diff --git a/solr/licenses/junit-4.11.jar.sha1 b/solr/licenses/junit-4.11.jar.sha1 new file mode 100644 index 000000000000..409d4c92e85d --- /dev/null +++ b/solr/licenses/junit-4.11.jar.sha1 @@ -0,0 +1 @@ +4e031bb61df09069aeb2bffb4019e7a5034a4ee0 diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java index 1dabe5de97c2..debefcdbed9a 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java @@ -69,7 +69,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.internal.matchers.StringContains.containsString; +import static org.hamcrest.Matchers.containsString; /** * This should include tests against the example solr config diff --git a/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java b/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java index 96b6d3132a0f..ec24fa0c6e26 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrIgnoredThreadsFilter.java @@ -19,6 +19,7 @@ import org.apache.lucene.search.TimeLimitingCollector.TimerThread; import com.carrotsearch.randomizedtesting.ThreadFilter; +import org.apache.solr.store.hdfs.HdfsLockFactory; /** @@ -60,6 +61,11 @@ public boolean reject(Thread t) { return true; } + if (threadName.startsWith(HdfsLockFactory.HDFS_LOCK_UPDATE_PREFIX)) { + return true; + } + + return false; } } diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 01e2cae4c680..80f8591448e4 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -126,6 +126,7 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.servlet.DirectSolrConnection; +import org.apache.solr.store.hdfs.HdfsLockFactory; import org.apache.solr.util.LogLevel; import org.apache.solr.util.RandomizeSSL; import org.apache.solr.util.RandomizeSSL.SSLRandomizer; @@ -308,6 +309,7 @@ public static void teardownTestCases() throws Exception { } resetFactory(); coreName = DEFAULT_TEST_CORENAME; + HdfsLockFactory.INSTANCE.stopBackgroundTasks(); } finally { ObjectReleaseTracker.clear(); TestInjection.reset(); @@ -349,7 +351,7 @@ public static void assumeWorkingMockito() { fail("ByteBuddy and Mockito are not available on classpath: " + e.toString()); } } - + /** * @return null if ok else error message */ @@ -810,7 +812,7 @@ public static void deleteCore() { // clears the updatelog sysprop at the end of the test run System.clearProperty(UPDATELOG_SYSPROP); } - + solrConfig = null; h = null; lrf = null; @@ -2269,27 +2271,27 @@ public static Object skewed(Object likely, Object unlikely) { * some internal settings. */ public static class CloudSolrClientBuilder extends CloudSolrClient.Builder { - + public CloudSolrClientBuilder(List zkHosts, Optional zkChroot) { super(zkHosts, zkChroot); randomizeCloudSolrClient(); } - + public CloudSolrClientBuilder(ClusterStateProvider stateProvider) { this.stateProvider = stateProvider; randomizeCloudSolrClient(); } - + public CloudSolrClientBuilder(MiniSolrCloudCluster cluster) { if (random().nextBoolean()) { this.zkHosts.add(cluster.getZkServer().getZkAddress()); } else { populateSolrUrls(cluster); } - + randomizeCloudSolrClient(); } - + private void populateSolrUrls(MiniSolrCloudCluster cluster) { if (random().nextBoolean()) { final List solrNodes = cluster.getJettySolrRunners(); @@ -2300,7 +2302,7 @@ private void populateSolrUrls(MiniSolrCloudCluster cluster) { this.solrUrls.add(cluster.getRandomJetty(random()).getBaseUrl().toString()); } } - + private void randomizeCloudSolrClient() { this.directUpdatesToLeadersOnly = random().nextBoolean(); this.shardLeadersOnly = random().nextBoolean();