HDDS-6614. EC: Fix Datanode block file INCONSISTENCY during heavy load.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLayoutVersion.java

@@ -142,7 +142,15 @@ private static File getChunkDir(ContainerData containerData)
       throw new StorageContainerException("Unable to get Chunks directory.",
           UNABLE_TO_FIND_DATA_DIR);
     }
-    return new File(chunksPath);
+
+    File chunksDir = new File(chunksPath);
+    if (!chunksDir.exists()) {
+      LOG.error("Chunks dir {} does not exist", chunksDir.getAbsolutePath());
+      throw new StorageContainerException("Chunks directory " +
+          chunksDir.getAbsolutePath() + " does not exist.",
+          UNABLE_TO_FIND_DATA_DIR);
+    }
+    return chunksDir;
   }
 
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java

@@ -383,14 +383,22 @@ private static ContainerProtos.Result translate(Exception cause) {
    * Checks if the block file length is equal to the chunk offset.
    *
    */
-  public static void validateChunkSize(File chunkFile, ChunkInfo chunkInfo)
+  public static void validateChunkSize(FileChannel fileChannel,
+      ChunkInfo chunkInfo, String fileName)
       throws StorageContainerException {
     long offset = chunkInfo.getOffset();
-    long len = chunkFile.length();
-    if (chunkFile.length() != offset) {
-      throw new StorageContainerException(
-          "Chunk file offset " + offset + " does not match blockFile length " +
-          len, CHUNK_FILE_INCONSISTENCY);
+    long fileLen;
+    try {
+      fileLen = fileChannel.size();
+    } catch (IOException e) {
+      throw new StorageContainerException("IO error encountered while " +
+          "getting the file size for " + fileName + " at offset " + offset,
+          CHUNK_FILE_INCONSISTENCY);
+    }
+    if (fileLen != offset) {
+      throw new StorageContainerException("Chunk offset " + offset +
+          " does not match length " + fileLen + " of blockFile " + fileName,
+          CHUNK_FILE_INCONSISTENCY);
     }
   }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java

@@ -137,7 +137,7 @@ public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
 
     // check whether offset matches block file length if its an overwrite
     if (!overwrite) {
-      ChunkUtils.validateChunkSize(chunkFile, info);
+      ChunkUtils.validateChunkSize(channel, info, chunkFile.getName());
     }
 
     ChunkUtils