Skip to content
Permalink
Browse files
HBASE-25331 [HBOSS] TreeLockManager should log warnings about lock co…
…ntention periodically (#18)

Signed-off-by: Peter Somogyi <psomogyi@apache.org>
  • Loading branch information
wchevreuil committed Nov 30, 2020
1 parent 5aa165f commit 64df29a2a8f4e710d3a1a90651eaf00163b22d9d
Showing 3 changed files with 39 additions and 6 deletions.
@@ -32,4 +32,6 @@ public class Constants {
public static final String ZK_MAX_RETRIES = "fs.hboss.sync.zk.sleep.max.retries";

public static final String CONTRACT_TEST_SCHEME = "fs.contract.test.fs.scheme";

public static final String WAIT_INTERVAL_WARN = "fs.hboss.lock-wait.interval.warning";
}
@@ -44,13 +44,18 @@
@InterfaceAudience.Private
@InterfaceStability.Unstable
public abstract class TreeLockManager {

private static final Logger LOG =
LoggerFactory.getLogger(TreeLockManager.class);

public enum Depth {
DIRECTORY, RECURSIVE
}

public static final long DEFAULT_WAIT_INTERVAL_WARN = 5000;

protected long waitIntervalWarn = DEFAULT_WAIT_INTERVAL_WARN;

public static synchronized TreeLockManager get(FileSystem fs)
throws IOException {
Configuration conf = fs.getConf();
@@ -257,12 +262,17 @@ private boolean retryBackoff(int retries) throws IOException {
*/
protected void treeWriteLock(Path path, Depth depth) throws IOException {
int outerRetries = 0;
long outerInitAttempt = System.currentTimeMillis();
do {
int innerRetries = 0;
long innerInitAttempt = System.currentTimeMillis();
do {
// If there's already a write-lock above or below us in the tree, wait for it to leave
if (writeLockAbove(path) || writeLockBelow(path, depth)) {
LOG.warn("Blocked on some parent write lock, waiting: {}", path);
if (this.warnIfAboveThreshold(innerInitAttempt,
"Blocked on some parent write lock, waiting: {}", path.toString())) {
innerInitAttempt = System.currentTimeMillis();
}
continue;
}
break;
@@ -271,7 +281,10 @@ protected void treeWriteLock(Path path, Depth depth) throws IOException {
writeLock(path);
// If there's now a write-lock above or below us in the tree, release and retry
if (writeLockAbove(path) || writeLockBelow(path, depth)) {
LOG.warn("Blocked on some other write lock, retrying: {}", path);
if (this.warnIfAboveThreshold(outerInitAttempt,
"Blocked on some other write lock, retrying: {}", path.toString())) {
outerInitAttempt = System.currentTimeMillis();
}
writeUnlock(path);
continue;
}
@@ -299,13 +312,17 @@ protected void treeWriteLock(Path path, Depth depth) throws IOException {
*/
protected void treeReadLock(Path path) throws IOException {
int outerRetries = 0;
long outerInitAttempt = System.currentTimeMillis();
do {
int innerRetries = 0;
long innerInitAttempt = System.currentTimeMillis();
do {
// If there's a write lock above us, wait
if (writeLockAbove(path)) {
LOG.warn("Blocked waiting for some parent write lock, waiting: {}",
path);
if (warnIfAboveThreshold(innerInitAttempt,
"Blocked waiting for some parent write lock, waiting: {}", path.toString())){
innerInitAttempt = System.currentTimeMillis();
}
continue;
}
break;
@@ -314,15 +331,27 @@ protected void treeReadLock(Path path) throws IOException {
readLock(path);
// If there's a write lock above us, release the lock and try again
if (writeLockAbove(path)) {
LOG.warn("Blocked waiting for some parent write lock, retrying: {}",
path);
if (warnIfAboveThreshold(outerInitAttempt,
"Blocked waiting for some parent write lock, retrying: {}", path.toString())){
outerInitAttempt = System.currentTimeMillis();
}
readUnlock(path);
continue;
}
break;
} while (retryBackoff(outerRetries++));
}

private boolean warnIfAboveThreshold(long start, String message, String... params) {
long blockedTime = System.currentTimeMillis() - start;
if (blockedTime >= this.waitIntervalWarn) {
String blockWarn = Thread.currentThread().getName() + " has been waiting on a lock for "
+ blockedTime + "ms. More details: \n";
LOG.warn(blockWarn + message, params);
return true;
}
return false;
}
/**
* Acquires an exclusive lock on a single path to create or append to a file.
* This is required for createNonRecursive() as well as other operations to be
@@ -78,6 +78,8 @@ public void initialize(FileSystem fs) throws IOException {
Configuration conf = fs.getConf();
int baseSleepTimeMs = conf.getInt(Constants.ZK_BASE_SLEEP_MS, 1000);
int maxRetries = conf.getInt(Constants.ZK_MAX_RETRIES, 3);
this.waitIntervalWarn = conf.getLong(Constants.WAIT_INTERVAL_WARN,
TreeLockManager.DEFAULT_WAIT_INTERVAL_WARN);
RetryPolicy retryPolicy = new ExponentialBackoffRetry(baseSleepTimeMs, maxRetries);

// Create a temporary connection to ensure the root is created, then create

0 comments on commit 64df29a

Please sign in to comment.