[ALLUXIO-3147] Enable more commands to use wildcards. (#7040)

* Enable more commands to use wildcards. Refactored wildcard command to enable more commands to use wildcard in their URIs. All commands are now children of AbstractFileSystemCommand. * addressing comments and some style changes. * changing the interface to include CommandLine. Removing unnecessary files. * fixing indent and using command line object.
Alluxio · Apr 10, 2018 · 489e0bc · 489e0bc
1 parent 41dc5c1
commit 489e0bc
Show file tree

Hide file tree

Showing 34 changed files with 495 additions and 171 deletions.
diff --git a/shell/src/main/java/alluxio/cli/fs/command/AbstractFileSystemCommand.java b/shell/src/main/java/alluxio/cli/fs/command/AbstractFileSystemCommand.java
@@ -11,10 +11,20 @@
 
 package alluxio.cli.fs.command;
 
+import alluxio.AlluxioURI;
 import alluxio.cli.Command;
+import alluxio.cli.fs.FileSystemShellUtils;
 import alluxio.client.file.FileSystem;
+import alluxio.exception.AlluxioException;
+
+import com.google.common.base.Joiner;
+import org.apache.commons.cli.CommandLine;
 
 import javax.annotation.concurrent.ThreadSafe;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
 
 /**
  * The base class for all the FileSystem {@link alluxio.cli.Command} classes.
@@ -28,4 +38,46 @@ public abstract class AbstractFileSystemCommand implements Command {
   protected AbstractFileSystemCommand(FileSystem fs) {
     mFileSystem = fs;
   }
+
+  /**
+   * Run the command for a particular URI that does not contain wildcard in its path.
+   *
+   * @param plainPath an AlluxioURI that does not contain wildcard
+   * @param cl object containing the original commandLine
+   * @throws AlluxioException
+   * @throws IOException
+   */
+  protected void runPlainPath(AlluxioURI plainPath, CommandLine cl)
+      throws AlluxioException, IOException {
+  }
+
+  /**
+   * Run the command for a particular URI that may contain wildcard in its path.
+   *
+   * @param wildCardPath an AlluxioURI that may or may not contain a wildcard
+   * @param cl object containing the original commandLine
+   * @throws AlluxioException
+   * @throws IOException
+   */
+  protected void runWildCardCmd(AlluxioURI wildCardPath, CommandLine cl) throws IOException {
+    List<AlluxioURI> paths = FileSystemShellUtils.getAlluxioURIs(mFileSystem, wildCardPath);
+    if (paths.size() == 0) { // A unified sanity check on the paths
+      throw new IOException(wildCardPath + " does not exist.");
+    }
+    paths.sort(Comparator.comparing(AlluxioURI::getPath));
+
+    List<String> errorMessages = new ArrayList<>();
+    for (AlluxioURI path : paths) {
+      try {
+        runPlainPath(path, cl);
+      } catch (AlluxioException | IOException e) {
+        errorMessages.add(e.getMessage() != null ? e.getMessage() : e.toString());
+      }
+    }
+
+    if (errorMessages.size() != 0) {
+      throw new IOException(Joiner.on('\n').join(errorMessages));
+    }
+
+  }
 }
diff --git a/shell/src/main/java/alluxio/cli/fs/command/CatCommand.java b/shell/src/main/java/alluxio/cli/fs/command/CatCommand.java
@@ -32,7 +32,7 @@
  * Prints the file's contents to the console.
  */
 @ThreadSafe
-public final class CatCommand extends WithWildCardPathCommand {
+public final class CatCommand extends AbstractFileSystemCommand {
 
   /**
    * @param fs the filesystem of Alluxio
@@ -47,7 +47,8 @@ public String getCommandName() {
   }
 
   @Override
-  protected void runCommand(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException {
+  protected void runPlainPath(AlluxioURI path, CommandLine cl)
+      throws AlluxioException, IOException {
     URIStatus status = mFileSystem.getStatus(path);
 
     if (status.isFolder()) {
@@ -78,4 +79,12 @@ public String getDescription() {
   public void validateArgs(CommandLine cl) throws InvalidArgumentException {
     CommandUtils.checkNumOfArgsNoLessThan(this, cl, 1);
   }
+
+  @Override
+  public int run(CommandLine cl) throws IOException {
+    String[] args = cl.getArgs();
+    runWildCardCmd(new AlluxioURI(args[0]), cl);
+
+    return 0;
+  }
 }
diff --git a/shell/src/main/java/alluxio/cli/fs/command/CheckConsistencyCommand.java b/shell/src/main/java/alluxio/cli/fs/command/CheckConsistencyCommand.java
@@ -49,6 +49,12 @@ public CheckConsistencyCommand(FileSystem fs) {
     super(fs);
   }
 
+  @Override
+  protected void runPlainPath(AlluxioURI plainPath, CommandLine cl)
+      throws AlluxioException, IOException {
+    checkConsistency(plainPath, cl.hasOption("r"));
+  }
+
   @Override
   public void validateArgs(CommandLine cl) throws InvalidArgumentException {
     CommandUtils.checkNumOfArgsEquals(this, cl, 1);
@@ -68,7 +74,8 @@ public String getCommandName() {
   public int run(CommandLine cl) throws AlluxioException, IOException {
     String[] args = cl.getArgs();
     AlluxioURI root = new AlluxioURI(args[0]);
-    checkConsistency(root, cl.hasOption("r"));
+
+    runWildCardCmd(root, cl);
     return 0;
   }
 

diff --git a/shell/src/main/java/alluxio/cli/fs/command/ChecksumCommand.java b/shell/src/main/java/alluxio/cli/fs/command/ChecksumCommand.java
@@ -43,6 +43,19 @@ public ChecksumCommand(FileSystem fs) {
     super(fs);
   }
 
+  @Override
+  protected void runPlainPath(AlluxioURI plainPath, CommandLine cl)
+      throws AlluxioException, IOException {
+    URIStatus status = mFileSystem.getStatus(plainPath);
+    if (status.isFolder()) {
+      throw new FileDoesNotExistException(
+          ExceptionMessage.PATH_MUST_BE_FILE.getMessage(plainPath.getPath()));
+    }
+
+    String str = calculateChecksum(plainPath);
+    System.out.println("md5sum: " + str + "\n");
+  }
+
   @Override
   public String getCommandName() {
     return "checksum";
@@ -56,13 +69,7 @@ public void validateArgs(CommandLine cl) throws InvalidArgumentException {
   @Override
   public int run(CommandLine cl) throws AlluxioException, IOException {
     String[] args = cl.getArgs();
-    AlluxioURI loc = new AlluxioURI(args[0]);
-    URIStatus status = mFileSystem.getStatus(loc);
-    if (status.isFolder()) {
-      throw new FileDoesNotExistException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(args[0]));
-    }
-    String str = calculateChecksum(loc);
-    System.out.println("md5sum: " + str);
+    runWildCardCmd(new AlluxioURI(args[0]), cl);
     return 0;
   }
 

diff --git a/shell/src/main/java/alluxio/cli/fs/command/ChgrpCommand.java b/shell/src/main/java/alluxio/cli/fs/command/ChgrpCommand.java
@@ -39,6 +39,8 @@ public final class ChgrpCommand extends AbstractFileSystemCommand {
           .desc("change group recursively")
           .build();
 
+  private String mGroup;
+
   /**
    * Creates a new instance of {@link ChgrpCommand}.
    *
@@ -78,12 +80,19 @@ private void chgrp(AlluxioURI path, String group, boolean recursive)
     System.out.println("Changed group of " + path + " to " + group);
   }
 
+  @Override
+  protected void runPlainPath(AlluxioURI path, CommandLine cl)
+      throws IOException, AlluxioException {
+    chgrp(path, mGroup, cl.hasOption("R"));
+  }
+
   @Override
   public int run(CommandLine cl) throws AlluxioException, IOException {
     String[] args = cl.getArgs();
-    String group = args[0];
+    mGroup = args[0];
     AlluxioURI path = new AlluxioURI(args[1]);
-    chgrp(path, group, cl.hasOption("R"));
+    runWildCardCmd(path, cl);
+
     return 0;
   }
 

diff --git a/shell/src/main/java/alluxio/cli/fs/command/ChmodCommand.java b/shell/src/main/java/alluxio/cli/fs/command/ChmodCommand.java
@@ -42,6 +42,7 @@ public final class ChmodCommand extends AbstractFileSystemCommand {
           .build();
 
   private final ModeParser mParser = new ModeParser();
+  private String mModeString = "";
 
   /**
    * Creates a new instance of {@link ChmodCommand}.
@@ -52,6 +53,12 @@ public ChmodCommand(FileSystem fs) {
     super(fs);
   }
 
+  @Override
+  protected void runPlainPath(AlluxioURI plainPath, CommandLine cl)
+      throws AlluxioException, IOException {
+    chmod(plainPath, mModeString, cl.hasOption("R"));
+  }
+
   @Override
   public String getCommandName() {
     return "chmod";
@@ -87,9 +94,10 @@ private void chmod(AlluxioURI path, String modeStr, boolean recursive) throws
   @Override
   public int run(CommandLine cl) throws AlluxioException, IOException {
     String[] args = cl.getArgs();
-    String modeStr = args[0];
+    mModeString = args[0];
+
     AlluxioURI path = new AlluxioURI(args[1]);
-    chmod(path, modeStr, cl.hasOption("R"));
+    runWildCardCmd(path, cl);
     return 0;
   }
 

diff --git a/shell/src/main/java/alluxio/cli/fs/command/ChownCommand.java b/shell/src/main/java/alluxio/cli/fs/command/ChownCommand.java
@@ -41,6 +41,9 @@ public final class ChownCommand extends AbstractFileSystemCommand {
           .desc("change owner recursively")
           .build();
 
+  private String mGroup;
+  private String mOwner;
+
   /**
    * Creates a new instance of {@link ChownCommand}.
    *
@@ -50,6 +53,16 @@ public ChownCommand(FileSystem fs) {
     super(fs);
   }
 
+  @Override
+  protected void runPlainPath(AlluxioURI path, CommandLine cl)
+      throws AlluxioException, IOException {
+    if (mGroup == null) {
+      chown(path, mOwner, cl.hasOption("R"));
+    } else {
+      chown(path, mOwner, mGroup, cl.hasOption("R"));
+    }
+  }
+
   @Override
   public String getCommandName() {
     return "chown";
@@ -110,13 +123,9 @@ public int run(CommandLine cl) throws AlluxioException, IOException {
     AlluxioURI path = new AlluxioURI(args[1]);
     Matcher matchUserGroup = USER_GROUP_PATTERN.matcher(args[0]);
     if (matchUserGroup.matches()) {
-      String owner = matchUserGroup.group("user");
-      String group = matchUserGroup.group("group");
-      if (group == null) {
-        chown(path, owner, cl.hasOption("R"));
-      } else {
-        chown(path, owner, group, cl.hasOption("R"));
-      }
+      mOwner = matchUserGroup.group("user");
+      mGroup = matchUserGroup.group("group");
+      runWildCardCmd(path, cl);
       return 0;
     }
     System.out.println("Failed to parse " + args[0] + " as user or user:group");

diff --git a/shell/src/main/java/alluxio/cli/fs/command/DuCommand.java b/shell/src/main/java/alluxio/cli/fs/command/DuCommand.java
@@ -29,7 +29,7 @@
  * Displays the size of a file or a directory specified by argv.
  */
 @ThreadSafe
-public final class DuCommand extends WithWildCardPathCommand {
+public final class DuCommand extends AbstractFileSystemCommand {
 
   /**
    * @param fs the filesystem of Alluxio
@@ -44,7 +44,8 @@ public String getCommandName() {
   }
 
   @Override
-  protected void runCommand(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException {
+  protected void runPlainPath(AlluxioURI path, CommandLine cl)
+      throws AlluxioException, IOException {
     long sizeInBytes = getFileOrFolderSize(mFileSystem, path);
     System.out.println(path + " is " + sizeInBytes + " bytes");
   }
@@ -85,4 +86,13 @@ public String getDescription() {
   public void validateArgs(CommandLine cl) throws InvalidArgumentException {
     CommandUtils.checkNumOfArgsEquals(this, cl, 1);
   }
+
+  @Override
+  public int run(CommandLine cl) throws AlluxioException, IOException {
+    String[] args = cl.getArgs();
+    AlluxioURI path = new AlluxioURI(args[0]);
+    runWildCardCmd(path, cl);
+
+    return 0;
+  }
 }
diff --git a/shell/src/main/java/alluxio/cli/fs/command/FileInfoCommand.java b/shell/src/main/java/alluxio/cli/fs/command/FileInfoCommand.java
@@ -11,16 +11,12 @@
 
 package alluxio.cli.fs.command;
 
-import alluxio.AlluxioURI;
 import alluxio.cli.CommandUtils;
 import alluxio.client.file.FileSystem;
-import alluxio.exception.AlluxioException;
 import alluxio.exception.status.InvalidArgumentException;
 
 import org.apache.commons.cli.CommandLine;
 
-import java.io.IOException;
-
 import javax.annotation.concurrent.ThreadSafe;
 
 /**
@@ -30,7 +26,7 @@
  */
 @ThreadSafe
 @Deprecated
-public final class FileInfoCommand extends WithWildCardPathCommand {
+public final class FileInfoCommand extends AbstractFileSystemCommand {
   /**
    * @param fs the filesystem of Alluxio
    */
@@ -44,10 +40,11 @@ public String getCommandName() {
   }
 
   @Override
-  protected void runCommand(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException {
+  public int run(CommandLine cl) {
     System.out
         .println("The \"alluxio fs fileInfo <path>\" command is deprecated since version 1.5.");
     System.out.println("Use the \"alluxio fs stat <path>\" command instead.");
+    return 0;
   }
 
   @Override

diff --git a/shell/src/main/java/alluxio/cli/fs/command/FreeCommand.java b/shell/src/main/java/alluxio/cli/fs/command/FreeCommand.java
@@ -31,7 +31,7 @@
  * folder).
  */
 @ThreadSafe
-public final class FreeCommand extends WithWildCardPathCommand {
+public final class FreeCommand extends AbstractFileSystemCommand {
 
   private static final Option FORCE_OPTION =
       Option.builder("f")
@@ -60,7 +60,8 @@ public String getCommandName() {
   }
 
   @Override
-  protected void runCommand(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException {
+  protected void runPlainPath(AlluxioURI path, CommandLine cl)
+      throws AlluxioException, IOException {
     FreeOptions options = FreeOptions.defaults().setRecursive(true).setForced(cl.hasOption("f"));
     mFileSystem.free(path, options);
     System.out.println(path + " was successfully freed from memory.");
@@ -77,6 +78,15 @@ public String getDescription() {
         + " Specify -f to force freeing pinned files in the directory.";
   }
 
+  @Override
+  public int run(CommandLine cl) throws AlluxioException, IOException {
+    String[] args = cl.getArgs();
+    AlluxioURI path = new AlluxioURI(args[0]);
+    runWildCardCmd(path, cl);
+
+    return 0;
+  }
+
   @Override
   public void validateArgs(CommandLine cl) throws InvalidArgumentException {
     CommandUtils.checkNumOfArgsNoLessThan(this, cl, 1);