Issue 4855: fail closed on entry log flush failure

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieImpl.java

@@ -500,6 +500,9 @@ public void ledgerDeleted(long ledgerId) {
         ledgerStorage.setStateManager(stateManager);
         ledgerStorage.setCheckpointSource(checkpointSource);
         ledgerStorage.setCheckpointer(syncThread);
+        if (isDbLedgerStorage) {
+            ((DbLedgerStorage) ledgerStorage).setFatalErrorListener(getLedgerDirsListener());
+        }
         ledgerStorage.registerLedgerDeletionListener(ledgerDeletionListener);
         handles = new HandleFactoryImpl(ledgerStorage);
 
@@ -1033,6 +1036,9 @@ public void recoveryAddEntry(ByteBuf entry, WriteCallback cb, Object ctx, byte[]
                 addEntryInternal(handle, entry, false /* ackBeforeSync */, cb, ctx, masterKey);
             }
             success = true;
+        } catch (EntryLogWriteException e) {
+            triggerBookieShutdown(ExitCode.BOOKIE_EXCEPTION);
+            throw e;
         } catch (NoWritableLedgerDirException e) {
             stateManager.transitionToReadOnlyMode();
             throw new IOException(e);
@@ -1125,6 +1131,9 @@ public void addEntry(ByteBuf entry, boolean ackBeforeSync, WriteCallback cb, Obj
                 addEntryInternal(handle, entry, ackBeforeSync, cb, ctx, masterKey);
             }
             success = true;
+        } catch (EntryLogWriteException e) {
+            triggerBookieShutdown(ExitCode.BOOKIE_EXCEPTION);
+            throw e;
         } catch (NoWritableLedgerDirException e) {
             stateManager.transitionToReadOnlyMode();
             throw new IOException(e);

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BufferedChannel.java

@@ -71,6 +71,7 @@ public class BufferedChannel extends BufferedReadChannel implements Closeable {
     protected final AtomicLong unpersistedBytes;
 
     private boolean closed = false;
+    private volatile IOException writeFailure;
 
     // make constructor to be public for unit test
     public BufferedChannel(ByteBufAllocator allocator, FileChannel fc, int capacity) throws IOException {
@@ -117,8 +118,14 @@ public synchronized void close() throws IOException {
     public void write(ByteBuf src) throws IOException {
         boolean shouldForceWrite = false;
         synchronized (this) {
-            int copied = copyIntoWriteBuffer(src);
-            shouldForceWrite = updatePositionAndFlushIfNeeded(copied);
+            checkWritable();
+            try {
+                int copied = copyIntoWriteBuffer(src);
+                shouldForceWrite = updatePositionAndFlushIfNeeded(copied);
+            } catch (IOException e) {
+                markWriteFailure(e);
+                throw e;
+            }
         }
         if (shouldForceWrite) {
             forceWrite(false);
@@ -137,9 +144,15 @@ public void write(ByteBuf src) throws IOException {
     public void write(ByteBuf src1, ByteBuf src2) throws IOException {
         boolean shouldForceWrite = false;
         synchronized (this) {
-            int copied = copyIntoWriteBuffer(src1);
-            copied += copyIntoWriteBuffer(src2);
-            shouldForceWrite = updatePositionAndFlushIfNeeded(copied);
+            checkWritable();
+            try {
+                int copied = copyIntoWriteBuffer(src1);
+                copied += copyIntoWriteBuffer(src2);
+                shouldForceWrite = updatePositionAndFlushIfNeeded(copied);
+            } catch (IOException e) {
+                markWriteFailure(e);
+                throw e;
+            }
         }
         if (shouldForceWrite) {
             forceWrite(false);
@@ -203,6 +216,7 @@ public long getFileChannelPosition() {
      * @throws IOException
      */
     public void flushAndForceWrite(boolean forceMetadata) throws IOException {
+        checkWritable();
         flush();
         forceWrite(forceMetadata);
     }
@@ -217,6 +231,7 @@ public void flushAndForceWrite(boolean forceMetadata) throws IOException {
      * @throws IOException
      */
     public void flushAndForceWriteIfRegularFlush(boolean forceMetadata) throws IOException {
+        checkWritable();
         if (doRegularFlushes) {
             flushAndForceWrite(forceMetadata);
         }
@@ -229,10 +244,19 @@ public void flushAndForceWriteIfRegularFlush(boolean forceMetadata) throws IOExc
      * @throws IOException if the write fails.
      */
     public synchronized void flush() throws IOException {
+        checkWritable();
         ByteBuffer toWrite = writeBuffer.internalNioBuffer(0, writeBuffer.writerIndex());
-        do {
-            fileChannel.write(toWrite);
-        } while (toWrite.hasRemaining());
+        try {
+            while (toWrite.hasRemaining()) {
+                int written = fileChannel.write(toWrite);
+                if (written <= 0) {
+                    throw new IOException("Unable to make progress while flushing buffered channel");
+                }
+            }
+        } catch (IOException e) {
+            markWriteFailure(e);
+            throw e;
+        }
         writeBuffer.clear();
         writeBufferStartPosition.set(fileChannel.position());
     }
@@ -244,6 +268,7 @@ public synchronized void flush() throws IOException {
      * @throws IOException
      */
     public long forceWrite(boolean forceMetadata) throws IOException {
+        checkWritable();
         // This is the point up to which we had flushed to the file system page cache
         // before issuing this force write hence is guaranteed to be made durable by
         // the force write, any flush that happens after this may or may
@@ -270,7 +295,12 @@ public long forceWrite(boolean forceMetadata) throws IOException {
             }
         }
 
-        fileChannel.force(forceMetadata);
+        try {
+            fileChannel.force(forceMetadata);
+        } catch (IOException e) {
+            markWriteFailure(e);
+            throw e;
+        }
         return positionForceWrite;
     }
 
@@ -333,4 +363,17 @@ public synchronized int getNumOfBytesInWriteBuffer() {
     long getUnpersistedBytes() {
         return unpersistedBytes.get();
     }
+
+    final void checkWritable() throws IOException {
+        IOException failure = writeFailure;
+        if (failure != null) {
+            throw new IOException("BufferedChannel is in failed state", failure);
+        }
+    }
+
+    final void markWriteFailure(IOException e) {
+        if (writeFailure == null) {
+            writeFailure = e;
+        }
+    }
 }

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/DefaultEntryLogger.java

@@ -56,8 +56,10 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Pattern;
 import lombok.CustomLog;
+import org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener;
 import org.apache.bookkeeper.bookie.storage.CompactionEntryLog;
 import org.apache.bookkeeper.bookie.storage.EntryLogScanner;
 import org.apache.bookkeeper.bookie.storage.EntryLogger;
@@ -136,6 +138,7 @@ public String toString() {
          * Updates the entry log file header with the offset and size of the map.
          */
         void appendLedgersMap() throws IOException {
+            checkWritable();
 
             long ledgerMapOffset = this.position();
 
@@ -204,7 +207,23 @@ public void accept(long ledgerId, long size) {
             mapInfo.putLong(ledgerMapOffset);
             mapInfo.putInt(numberOfLedgers);
             mapInfo.flip();
-            this.fileChannel.write(mapInfo, LEDGERS_MAP_OFFSET_POSITION);
+            try {
+                writeFully(this.fileChannel, mapInfo, LEDGERS_MAP_OFFSET_POSITION);
+            } catch (IOException e) {
+                markWriteFailure(e);
+                throw e;
+            }
+        }
+
+        private static void writeFully(FileChannel fileChannel, ByteBuffer buffer, long position) throws IOException {
+            long writePosition = position;
+            while (buffer.hasRemaining()) {
+                int written = fileChannel.write(buffer, writePosition);
+                if (written <= 0) {
+                    throw new IOException("Unable to make progress while updating entry log header");
+                }
+                writePosition += written;
+            }
         }
     }
 
@@ -222,6 +241,7 @@ public void accept(long ledgerId, long size) {
 
     final EntryLoggerAllocator entryLoggerAllocator;
     private final EntryLogManager entryLogManager;
+    private final AtomicBoolean closed = new AtomicBoolean(false);
 
     private final CopyOnWriteArrayList<EntryLogListener> listeners = new CopyOnWriteArrayList<EntryLogListener>();
 
@@ -365,6 +385,10 @@ EntryLogManager getEntryLogManager() {
         return entryLogManager;
     }
 
+    public void setFatalErrorListener(LedgerDirsListener fatalErrorListener) {
+        entryLogManager.setFatalErrorListener(fatalErrorListener);
+    }
+
     void addListener(EntryLogListener listener) {
         if (null != listener) {
             listeners.add(listener);
@@ -1208,6 +1232,10 @@ public boolean accept(long ledgerId) {
      */
     @Override
     public void close() {
+        if (!closed.compareAndSet(false, true)) {
+            log.debug("EntryLogger is already stopped");
+            return;
+        }
         // since logChannel is buffered channel, do flush when shutting down
         log.info("Stopping EntryLogger");
         try {

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogManager.java

@@ -26,6 +26,7 @@
 import java.io.IOException;
 import java.util.List;
 import org.apache.bookkeeper.bookie.DefaultEntryLogger.BufferedLogChannel;
+import org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener;
 
 interface EntryLogManager {
 
@@ -66,6 +67,11 @@ interface EntryLogManager {
      */
     void forceClose();
 
+    /*
+     * notify the owning bookie when entry-log-level writes become fatal.
+     */
+    void setFatalErrorListener(LedgerDirsListener fatalErrorListener);
+
     /*
      * prepare entrylogger/entrylogmanager before doing SortedLedgerStorage
      * Checkpoint.

bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/EntryLogManagerBase.java

@@ -33,6 +33,7 @@
 import lombok.CustomLog;
 import org.apache.bookkeeper.bookie.DefaultEntryLogger.BufferedLogChannel;
 import org.apache.bookkeeper.bookie.DefaultEntryLogger.EntryLogListener;
+import org.apache.bookkeeper.bookie.LedgerDirsManager.LedgerDirsListener;
 import org.apache.bookkeeper.bookie.LedgerDirsManager.NoWritableLedgerDirException;
 import org.apache.bookkeeper.conf.ServerConfiguration;
 
@@ -42,6 +43,8 @@ abstract class EntryLogManagerBase implements EntryLogManager {
     final EntryLoggerAllocator entryLoggerAllocator;
     final LedgerDirsManager ledgerDirsManager;
     private final List<DefaultEntryLogger.EntryLogListener> listeners;
+    private static final LedgerDirsListener NOOP_FATAL_ERROR_LISTENER = new LedgerDirsListener() { };
+    private volatile LedgerDirsListener fatalErrorListener = NOOP_FATAL_ERROR_LISTENER;
     /**
      * The maximum size of a entry logger file.
      */
@@ -73,13 +76,20 @@ public long addEntry(long ledger, ByteBuf entry, boolean rollLog) throws IOExcep
         ByteBuf sizeBuffer = sizeBufferForAdd.get();
         sizeBuffer.clear();
         sizeBuffer.writeInt(entry.readableBytes());
-        logChannel.write(sizeBuffer);
-
-        long pos = logChannel.position();
-        logChannel.write(entry);
-        logChannel.registerWrittenEntry(ledger, entrySize);
-
-        return (logChannel.getLogId() << 32L) | pos;
+        try {
+            logChannel.write(sizeBuffer);
+
+            long pos = logChannel.position();
+            logChannel.write(entry);
+            logChannel.registerWrittenEntry(ledger, entrySize);
+
+            return (logChannel.getLogId() << 32L) | pos;
+        } catch (EntryLogWriteException e) {
+            throw e;
+        } catch (IOException e) {
+            throw new EntryLogWriteException(
+                    "Failed to write entry to entry log " + logChannel.getLogId() + " for ledger " + ledger, e);
+        }
     }
 
     boolean reachEntryLogLimit(BufferedLogChannel logChannel, long size) {
@@ -117,6 +127,21 @@ List<BufferedLogChannel> getRotatedLogChannels() {
         return rotatedLogChannels;
     }
 
+    @Override
+    public void setFatalErrorListener(LedgerDirsListener fatalErrorListener) {
+        this.fatalErrorListener = fatalErrorListener != null ? fatalErrorListener : NOOP_FATAL_ERROR_LISTENER;
+    }
+
+    void notifyFatalEntryLogWriteFailure(String message, Throwable cause) {
+        log.error().exception(cause).log(message);
+        fatalErrorListener.fatalError();
+        for (LedgerDirsListener listener : ledgerDirsManager.getListeners()) {
+            if (listener != fatalErrorListener) {
+                listener.fatalError();
+            }
+        }
+    }
+
     @Override
     public void flush() throws IOException {
         flushCurrentLogs();
@@ -125,11 +150,31 @@ public void flush() throws IOException {
 
     void flushLogChannel(BufferedLogChannel logChannel, boolean forceMetadata) throws IOException {
         if (logChannel != null) {
-            logChannel.flushAndForceWrite(forceMetadata);
+            flushAndForceWrite(logChannel, forceMetadata);
             log.debug().attr("logId", () -> logChannel.getLogId()).log("Flush and sync current entry logger");
         }
     }
 
+    void flushAndForceWrite(BufferedLogChannel logChannel, boolean forceMetadata) throws IOException {
+        try {
+            logChannel.flushAndForceWrite(forceMetadata);
+        } catch (EntryLogWriteException e) {
+            throw e;
+        } catch (IOException e) {
+            throw new EntryLogWriteException("Failed to flush entry log " + logChannel.getLogId(), e);
+        }
+    }
+
+    void flushAndForceWriteIfRegularFlush(BufferedLogChannel logChannel, boolean forceMetadata) throws IOException {
+        try {
+            logChannel.flushAndForceWriteIfRegularFlush(forceMetadata);
+        } catch (EntryLogWriteException e) {
+            throw e;
+        } catch (IOException e) {
+            throw new EntryLogWriteException("Failed to flush entry log " + logChannel.getLogId(), e);
+        }
+    }
+
     /*
      * Creates a new log file. This method should be guarded by a lock,
      * so callers of this method should be in right scope of the lock.
@@ -156,12 +201,27 @@ void createNewLog(long ledgerId, String reason) throws IOException {
         if (null != logChannel) {
 
             // flush the internal buffer back to filesystem but not sync disk
-            logChannel.flush();
-
-            // Append ledgers map at the end of entry log
-            logChannel.appendLedgersMap();
+            try {
+                logChannel.flush();
+
+                // Append ledgers map at the end of entry log
+                logChannel.appendLedgersMap();
+            } catch (EntryLogWriteException e) {
+                throw e;
+            } catch (IOException e) {
+                throw new EntryLogWriteException(
+                        "Failed to rotate entry log " + logChannel.getLogId() + " for ledger " + ledgerId, e);
+            }
 
-            BufferedLogChannel newLogChannel = entryLoggerAllocator.createNewLog(selectDirForNextEntryLog());
+            File dirForNextEntryLog = selectDirForNextEntryLog();
+            BufferedLogChannel newLogChannel;
+            try {
+                newLogChannel = entryLoggerAllocator.createNewLog(dirForNextEntryLog);
+            } catch (EntryLogWriteException e) {
+                throw e;
+            } catch (IOException e) {
+                throw new EntryLogWriteException("Failed to create a new entry log for ledger " + ledgerId, e);
+            }
             entryLoggerAllocator.setWritingLogId(newLogChannel.getLogId());
             setCurrentLogForLedgerAndAddToRotate(ledgerId, newLogChannel);
             log.info()
@@ -172,7 +232,15 @@ void createNewLog(long ledgerId, String reason) throws IOException {
                 listener.onRotateEntryLog();
             }
         } else {
-            BufferedLogChannel newLogChannel = entryLoggerAllocator.createNewLog(selectDirForNextEntryLog());
+            File dirForNextEntryLog = selectDirForNextEntryLog();
+            BufferedLogChannel newLogChannel;
+            try {
+                newLogChannel = entryLoggerAllocator.createNewLog(dirForNextEntryLog);
+            } catch (EntryLogWriteException e) {
+                throw e;
+            } catch (IOException e) {
+                throw new EntryLogWriteException("Failed to create a new entry log for ledger " + ledgerId, e);
+            }
             entryLoggerAllocator.setWritingLogId(newLogChannel.getLogId());
             setCurrentLogForLedgerAndAddToRotate(ledgerId, newLogChannel);
         }