HDFS-17046. Delete path directly when it can not be parsed in trash.

apache · Jun 14, 2023 · 078150c · 078150c
1 parent f0c4286
commit 078150c
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 9 deletions.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java
@@ -137,7 +137,7 @@ public boolean isEnabled() {
    * @throws IOException raised on errors performing I/O.
    */ 
   public boolean moveToTrash(Path path) throws IOException {
-    return trashPolicy.moveToTrash(path);
+    return trashPolicy.moveToTrash(path, false);
   }
 
   /**

diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
@@ -68,10 +68,11 @@ public void initialize(Configuration conf, FileSystem fs) {
   /** 
    * Move a file or directory to the current trash directory.
    * @param path the path.
+   * @param force Whether force moving or not.
    * @return false if the item is already in the trash or trash is disabled
    * @throws IOException raised on errors performing I/O.
-   */ 
-  public abstract boolean moveToTrash(Path path) throws IOException;
+   */
+  public abstract boolean moveToTrash(Path path, boolean force) throws IOException;
 
   /** 
    * Create a trash checkpoint.

diff --git a/...p-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/...p-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -120,7 +120,7 @@ public boolean isEnabled() {
 
   @SuppressWarnings("deprecation")
   @Override
-  public boolean moveToTrash(Path path) throws IOException {
+  public boolean moveToTrash(Path path, boolean force) throws IOException {
     if (!isEnabled())
       return false;
 
@@ -133,7 +133,7 @@ public boolean moveToTrash(Path path) throws IOException {
 
     Path trashRoot = fs.getTrashRoot(path);
     Path trashCurrent = new Path(trashRoot, CURRENT);
-    if (qpath.startsWith(trashRoot.toString())) {
+    if (qpath.startsWith(trashRoot.toString()) && !force) {
       return false;                               // already in trash
     }
 
@@ -374,7 +374,8 @@ private void deleteCheckpoint(Path trashRoot, boolean deleteImmediately)
       try {
         time = getTimeFromCheckpoint(name);
       } catch (ParseException e) {
-        LOG.warn("Unexpected item in trash: "+dir+". Ignoring.");
+        this.moveToTrash(path, true);
+        LOG.warn("Unexpected item in trash: " + dir + ". Force moving to trash.");
         continue;
       }
 

diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -786,6 +786,68 @@ public void testTrashEmptier() throws Exception {
     emptierThread.join();
   }
 
+  /**
+   * Test trash emptier can whether delete non-checkpoint dir or not.
+   * @throws Exception
+   */
+  @Test
+  public void testTrashEmptierWithNonCheckpointDir() throws Exception {
+    Configuration conf = new Configuration();
+    // Trash with 12 second deletes and 6 seconds checkpoints
+    conf.set(FS_TRASH_INTERVAL_KEY, "0.2"); // 12 seconds
+    conf.setClass("fs.file.impl", TestLFS.class, FileSystem.class);
+    conf.set(FS_TRASH_CHECKPOINT_INTERVAL_KEY, "0.1"); // 6 seconds
+    FileSystem fs = FileSystem.getLocal(conf);
+    conf.set("fs.default.name", fs.getUri().toString());
+
+    Trash trash = new Trash(conf);
+
+    // Start Emptier in background
+    Runnable emptier = trash.getEmptier();
+    Thread emptierThread = new Thread(emptier);
+    emptierThread.start();
+
+    FsShell shell = new FsShell();
+    shell.setConf(conf);
+    shell.init();
+
+    // First create a new directory with mkdirs
+    Path myPath = new Path(TEST_DIR, "test/mkdirs");
+    mkdir(fs, myPath);
+    // Make sure the .Trash dir existed.
+    mkdir(fs, shell.getCurrentTrashDir());
+    assertTrue(fs.exists(shell.getCurrentTrashDir()));
+
+    int fileIndex = 0;
+    int loopCount = 10;
+    while (fileIndex < loopCount) {
+      // Create a file with a new name
+      Path myFile = new Path(TEST_DIR, "test/mkdirs/myFile" + fileIndex++);
+      writeFile(fs, myFile, 10);
+
+      // Move the file to trash root.
+      String[] args = new String[3];
+      args[0] = "-mv";
+      args[1] = myFile.toString();
+      args[2] = shell.getCurrentTrashDir().getParent().toString();
+      int val = -1;
+      try {
+        val = shell.run(args);
+      } catch (Exception e) {
+        System.err.println("Exception raised from Trash.run " +
+            e.getLocalizedMessage());
+      }
+      assertTrue(val == 0);
+    }
+    Thread.sleep(18000);
+
+    Path trashDir = shell.getCurrentTrashDir();
+    FileStatus files[] = fs.listStatus(trashDir.getParent());
+    assertTrue(files.length <= 1);
+    emptierThread.interrupt();
+    emptierThread.join();
+  }
+
   @After
   public void tearDown() throws IOException {
     File trashDir = new File(TEST_DIR.toUri().getPath());
@@ -1059,8 +1121,8 @@ public boolean isEnabled() {
       return false;
     }
 
-    @Override 
-    public boolean moveToTrash(Path path) throws IOException {
+    @Override
+    public boolean moveToTrash(Path path, boolean force) throws IOException {
       return false;
     }
 
@@ -1121,7 +1183,7 @@ public void initialize(Configuration conf, FileSystem fs) {
     }
 
     @Override
-    public boolean moveToTrash(Path path) throws IOException {
+    public boolean moveToTrash(Path path, boolean force) throws IOException {
       return false;
     }