Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve logging on ledger dirs monitor to avoid log flooding #1322

Closed
wants to merge 1 commit into from
Closed

Improve logging on ledger dirs monitor to avoid log flooding #1322

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 7 additions & 0 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieStateManager.java
View file
Open in desktop
Expand Up @@ -31,6 +31,7 @@
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.meta.MetadataBookieDriver;
import org.apache.bookkeeper.stats.Gauge;
Expand All @@ -43,6 +44,7 @@
/**
* An implementation of StateManager.
*/
@Slf4j
public class BookieStateManager implements StateManager {
private static final Logger LOG = LoggerFactory.getLogger(BookieStateManager.class);
private final ServerConfiguration conf;
Expand Down Expand Up @@ -142,6 +144,11 @@ public boolean isAvailableForHighPriorityWrites() {

@Override
public void setHighPriorityWritesAvailability(boolean available) {
if (this.availableForHighPriorityWrites && !available) {
log.info("Disable high priority writes on readonly bookie.");
} else if (!this.availableForHighPriorityWrites && available) {
log.info("Enable high priority writes on readonly bookie.");
}
this.availableForHighPriorityWrites = available;
}

Expand Down
15 changes: 10 additions & 5 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsManager.java
View file
Open in desktop
Expand Up @@ -179,10 +179,11 @@ public List<File> getWritableLedgerDirsForNewLog() throws NoWritableLedgerDirExc
// We don't have writable Ledger Dirs. But we are still okay to create new entry log files if we have enough
// disk spaces. This allows bookie can still function at readonly mode. Because compaction, journal replays
// can still write data to disks.
return getDirsAboveUsableThresholdSize(minUsableSizeForEntryLogCreation);
return getDirsAboveUsableThresholdSize(minUsableSizeForEntryLogCreation, true);
}

List<File> getDirsAboveUsableThresholdSize(long thresholdSize) throws NoWritableLedgerDirException {
List<File> getDirsAboveUsableThresholdSize(long thresholdSize, boolean loggingNoWritable)
throws NoWritableLedgerDirException {
List<File> fullLedgerDirsToAccomodate = new ArrayList<File>();
for (File dir: this.ledgerDirectories) {
// Pick dirs which can accommodate little more than thresholdSize
Expand All @@ -192,16 +193,20 @@ List<File> getDirsAboveUsableThresholdSize(long thresholdSize) throws NoWritable
}

if (!fullLedgerDirsToAccomodate.isEmpty()) {
LOG.info("No writable ledger dirs below diskUsageThreshold. "
if (loggingNoWritable) {
LOG.info("No writable ledger dirs below diskUsageThreshold. "
+ "But Dirs that can accommodate {} are: {}", thresholdSize, fullLedgerDirsToAccomodate);
}
return fullLedgerDirsToAccomodate;
}

// We will reach here when we find no ledgerDir which has atleast
// thresholdSize usable space
String errMsg = "All ledger directories are non writable and no reserved space (" + thresholdSize + ") left.";
NoWritableLedgerDirException e = new NoWritableLedgerDirException(errMsg);
LOG.error(errMsg, e);
if (loggingNoWritable) {
LOG.error(errMsg, e);
}
throw e;
}

Expand Down Expand Up @@ -307,7 +312,7 @@ File pickRandomWritableDirForNewIndexFile(File excludedDir) throws NoWritableLed
// That means we must have turned readonly. But
// during the Bookie restart, while replaying the journal there might be a need
// to create new Index file and it should proceed.
writableDirsForNewIndexFile = getDirsAboveUsableThresholdSize(minUsableSizeForIndexFileCreation);
writableDirsForNewIndexFile = getDirsAboveUsableThresholdSize(minUsableSizeForIndexFileCreation, true);
}
return pickRandomDir(writableDirsForNewIndexFile, excludedDir);
}
Expand Down
19 changes: 14 additions & 5 deletions bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDirsMonitor.java
View file
Open in desktop
Expand Up @@ -82,14 +82,22 @@ private void check() {
listener.diskFailed(dir);
}
} catch (DiskWarnThresholdException e) {
LOG.warn("Ledger directory {} is almost full.", dir);
diskUsages.put(dir, e.getUsage());
diskUsages.compute(dir, (d, prevUsage) -> {
if (null == prevUsage || e.getUsage() != prevUsage) {
LOG.warn("Ledger directory {} is almost full : usage {}", dir, e.getUsage());
}
return e.getUsage();
});
for (LedgerDirsListener listener : ldm.getListeners()) {
listener.diskAlmostFull(dir);
}
} catch (DiskOutOfSpaceException e) {
LOG.error("Ledger directory {} is out-of-space.", dir);
diskUsages.put(dir, e.getUsage());
diskUsages.compute(dir, (d, prevUsage) -> {
if (null == prevUsage || e.getUsage() != prevUsage) {
LOG.error("Ledger directory {} is out-of-space : usage {}", dir, e.getUsage());
}
return e.getUsage();
});
// Notify disk full to all listeners
ldm.addToFilledDirs(dir);
}
Expand All @@ -102,7 +110,8 @@ private void check() {
} catch (NoWritableLedgerDirException e) {
boolean highPriorityWritesAllowed = true;
try {
ldm.getDirsAboveUsableThresholdSize(minUsableSizeForHighPriorityWrites);
// disk check can be frequent, so disable 'loggingNoWritable' to avoid log flooding.
ldm.getDirsAboveUsableThresholdSize(minUsableSizeForHighPriorityWrites, false);
} catch (NoWritableLedgerDirException e1) {
highPriorityWritesAllowed = false;
}
Expand Down