HDDS-2372. Datanode pipeline is failing with NoSuchFileException (#166)

apache · Nov 27, 2019 · fe7fccf · fe7fccf
1 parent 4f36984
commit fe7fccf
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 51 deletions.
diff --git a/...-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java b/...-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/ChunkUtils.java
@@ -18,30 +18,12 @@
 
 package org.apache.hadoop.ozone.container.keyvalue.helpers;
 
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.scm.container.common.helpers
-    .StorageContainerException;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats;
-import org.apache.hadoop.util.Time;
-import org.apache.ratis.util.function.CheckedSupplier;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
+import java.nio.file.NoSuchFileException;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
@@ -52,8 +34,28 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 
-import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.*;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
+import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.util.Time;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import static java.util.Collections.unmodifiableSet;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.CONTAINER_INTERNAL_ERROR;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.INVALID_WRITE_SIZE;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.IO_EXCEPTION;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNABLE_TO_FIND_CHUNK;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNABLE_TO_FIND_DATA_DIR;
+import org.apache.ratis.util.function.CheckedSupplier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Utility methods for chunk operations for KeyValue container.
@@ -163,14 +165,6 @@ public static int validateBufferSize(
   public static ByteBuffer readData(File chunkFile, ChunkInfo data,
       VolumeIOStats volumeIOStats) throws StorageContainerException {
 
-    if (!chunkFile.exists()) {
-      LOG.error("Unable to find the chunk file. chunk info : {}",
-          data.toString());
-      throw new StorageContainerException("Unable to find the chunk file. " +
-          "chunk info " +
-          data.toString(), UNABLE_TO_FIND_CHUNK);
-    }
-
     long offset = data.getOffset();
     long len = data.getLen();
     ByteBuffer buf = ByteBuffer.allocate((int) len);
@@ -194,6 +188,8 @@ public static ByteBuffer readData(File chunkFile, ChunkInfo data,
         volumeIOStats.incReadBytes(len);
 
         return buf;
+      } catch (NoSuchFileException e) {
+        throw new StorageContainerException(e, UNABLE_TO_FIND_CHUNK);
       } catch (IOException e) {
         throw new StorageContainerException(e, IO_EXCEPTION);
       } finally {

diff --git a/...rvice/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/...rvice/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.ozone.container.keyvalue.impl;
 
 import com.google.common.base.Preconditions;
+
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
@@ -32,6 +33,8 @@
 import org.apache.hadoop.ozone.container.common.impl.ChunkLayOutVersion;
 import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
 import org.apache.hadoop.ozone.container.common.interfaces.Container;
+
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNABLE_TO_FIND_CHUNK;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,6 +44,8 @@
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
 import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.ExecutionException;
 
 import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
@@ -213,19 +218,35 @@ public ByteBuffer readChunk(Container container, BlockID blockID,
     // of the chunk file.
     if (containerData.getLayOutVersion() == ChunkLayOutVersion
         .getLatestVersion().getVersion()) {
-      File chunkFile = ChunkUtils.getChunkFile(containerData, info);
 
-      // In case the chunk file does not exist but tmp chunk file exist,
-      // read from tmp chunk file if readFromTmpFile is set to true
-      if (!chunkFile.exists() && dispatcherContext != null
-          && dispatcherContext.isReadFromTmpFile()) {
-        chunkFile = getTmpChunkFile(chunkFile, dispatcherContext);
+      File finalChunkFile = ChunkUtils.getChunkFile(containerData, info);
+
+      List<File> possibleFiles = new ArrayList<>();
+      possibleFiles.add(finalChunkFile);
+      if (dispatcherContext != null && dispatcherContext.isReadFromTmpFile()) {
+        possibleFiles.add(getTmpChunkFile(finalChunkFile, dispatcherContext));
+        possibleFiles.add(finalChunkFile);
       }
-      data = ChunkUtils.readData(chunkFile, info, volumeIOStats);
-      containerData.incrReadCount();
-      long length = chunkFile.length();
-      containerData.incrReadBytes(length);
-      return data;
+
+      for (File chunkFile : possibleFiles) {
+        try {
+          data = ChunkUtils.readData(chunkFile, info, volumeIOStats);
+          containerData.incrReadCount();
+          long length = info.getLen();
+          containerData.incrReadBytes(length);
+          return data;
+        } catch (StorageContainerException ex) {
+          //UNABLE TO FIND chunk is not a problem as we will try with the
+          //next possible location
+          if (ex.getResult() != UNABLE_TO_FIND_CHUNK) {
+            throw ex;
+          }
+        }
+      }
+      throw new StorageContainerException(
+          "Chunk file can't be found " + possibleFiles.toString(),
+          UNABLE_TO_FIND_CHUNK);
+
     }
     return null;
   }

diff --git a/...ervice/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/...ervice/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java
@@ -237,8 +237,7 @@ public void testReadChunkFileNotExists() throws Exception {
           blockID, chunkInfo, getDispatcherContext());
       fail("testReadChunkFileNotExists failed");
     } catch (StorageContainerException ex) {
-      GenericTestUtils.assertExceptionContains("Unable to find the chunk " +
-          "file.", ex);
+      GenericTestUtils.assertExceptionContains("Chunk file can't be found", ex);
       assertEquals(ContainerProtos.Result.UNABLE_TO_FIND_CHUNK, ex.getResult());
     }
   }

diff --git a/...vice/src/test/java/org/apache/hadoop/ozone/container/keyvalue/helpers/TestChunkUtils.java b/...vice/src/test/java/org/apache/hadoop/ozone/container/keyvalue/helpers/TestChunkUtils.java
@@ -17,16 +17,6 @@
  */
 package org.apache.hadoop.ozone.container.keyvalue.helpers;
 
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats;
-import org.apache.hadoop.test.GenericTestUtils;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -43,9 +33,21 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats;
+import org.apache.hadoop.test.GenericTestUtils;
+
+import org.apache.commons.io.FileUtils;
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.UNABLE_TO_FIND_CHUNK;
+import org.junit.Assert;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Tests for {@link ChunkUtils}.
@@ -180,4 +182,17 @@ public void validateChunkForOverwrite() throws IOException {
             new ChunkInfo("chunk", 5, 5)));
   }
 
+  @Test
+  public void readMissingFile() throws Exception {
+    try {
+      ChunkInfo chunkInfo =
+          new ChunkInfo("chunk_name", 0, 123);
+      ChunkUtils
+          .readData(new File("nosuchfile"), chunkInfo, new VolumeIOStats());
+      fail("Exception is Expected");
+    } catch (StorageContainerException ex) {
+      Assert.assertEquals(UNABLE_TO_FIND_CHUNK, ex.getResult());
+    }
+  }
+
 }