Add SSH and SCP command utility and refactor shell utility

core/common/src/main/java/alluxio/shell/CommandReturn.java

@@ -0,0 +1,60 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package alluxio.shell;
+
+/**
+ * Object representation of a command execution.
+ */
+public class CommandReturn {
+  private int mExitCode;
+  private String mOutput;
+
+  /**
+   * Creates object from the contents.
+   *
+   * @param code exit code
+   * @param output stdout content
+   */
+  public CommandReturn(int code, String output) {
+    mExitCode = code;
+    mOutput = output;
+  }
+
+  /**
+   * Gets the stdout content.
+   *
+   * @return stdout content
+   */
+  public String getOutput() {
+    return mOutput;
+  }
+
+  /**
+   * Gets the exit code.
+   *
+   * @return exit code of execution
+   */
+  public int getExitCode() {
+    return mExitCode;
+  }
+
+  /**
+   * Formats the object to more readable format.
+   * This is not done in toString() because stdout and stderr may be long.
+   *
+   * @return pretty formatted output
+   */
+  public String getFormattedOutput() {
+    return String.format("StatusCode:%s%nOutput:%n%s", getExitCode(),
+            getOutput());
+  }
+}

core/common/src/main/java/alluxio/shell/ScpCommand.java

@@ -0,0 +1,61 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package alluxio.shell;
+
+import alluxio.util.ShellUtils;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * Object representation of a remote scp command.
+ * The scp command copies a file/dir from remote host to local.
+ */
+@NotThreadSafe
+public class ScpCommand extends ShellCommand {
+  private final String mHostName;
+
+  /**
+   * Creates a remote scp command to copy a file to local.
+   *
+   * @param remoteHost remote hostname
+   * @param fromFile the remote file
+   * @param toFile target location to copy to
+   */
+  public ScpCommand(String remoteHost, String fromFile, String toFile) {
+    this(remoteHost, fromFile, toFile, false);
+  }
+
+  /**
+   * Creates a remote scp command to copy a file/dir to local.
+   *
+   * @param remoteHost the remote hostname
+   * @param fromFile the remote file/dir to copy
+   * @param toFile target path to copy to
+   * @param isDir if true, the remote file is a directory
+   */
+  public ScpCommand(String remoteHost, String fromFile, String toFile, boolean isDir) {
+    super(new String[]{"bash", "-c",
+            String.format(isDir ? "scp -r %s %s:%s localhost:%s" : "scp %s %s:%s localhost:%s",
+                    ShellUtils.COMMON_SSH_OPTS, remoteHost, fromFile, toFile)
+    });
+    mHostName = remoteHost;
+  }
+
+  /**
+   * Returns the remote target hostname.
+   *
+   * @return target hostname
+   */
+  public String getHostName() {
+    return mHostName;
+  }
+}

core/common/src/main/java/alluxio/shell/ShellCommand.java

@@ -0,0 +1,145 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package alluxio.shell;
+
+import alluxio.util.ShellUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.NotThreadSafe;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+/**
+ * Object representation of a shell command.
+ */
+@NotThreadSafe
+public class ShellCommand {
+  private static final Logger LOG = LoggerFactory.getLogger(ShellCommand.class);
+
+  private final String[] mCommand;
+
+  /**
+   * Creates a ShellCommand object with the command to exec.
+   *
+   * @param execString shell command
+   */
+  public ShellCommand(String[] execString) {
+    mCommand = execString.clone();
+  }
+
+  /**
+   * Runs a command and returns its stdout on success.
+   * Stderr is redirected to stdout.
+   *
+   * @return the output
+   * @throws IOException if the command returns a non-zero exit code
+   */
+  public String run() throws IOException {
+    Process process = new ProcessBuilder(mCommand).redirectErrorStream(true).start();
+
+    BufferedReader inReader =
+            new BufferedReader(new InputStreamReader(process.getInputStream(),
+                    Charset.defaultCharset()));
+
+    try {
+      // read the output of the command
+      StringBuilder output = new StringBuilder();
+      String line = inReader.readLine();
+      while (line != null) {
+        output.append(line);
+        output.append("\n");
+        line = inReader.readLine();
+      }
+      // wait for the process to finish and check the exit code
+      int exitCode = process.waitFor();
+      if (exitCode != 0) {
+        throw new ShellUtils.ExitCodeException(exitCode, output.toString());
+      }
+      return output.toString();
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new IOException(e);
+    } finally {
+      // close the input stream
+      try {
+        // JDK 7 tries to automatically drain the input streams for us
+        // when the process exits, but since close is not synchronized,
+        // it creates a race if we close the stream first and the same
+        // fd is recycled. the stream draining thread will attempt to
+        // drain that fd!! it may block, OOM, or cause bizarre behavior
+        // see: https://bugs.openjdk.java.net/browse/JDK-8024521
+        // issue is fixed in build 7u60
+        InputStream stdout = process.getInputStream();
+        synchronized (stdout) {
+          inReader.close();
+        }
+      } catch (IOException e) {
+        LOG.warn(String.format("Error while closing the input stream of process %s: %s",
+                process, e.getMessage()));
+      }
+      process.destroy();
+    }
+  }
+
+  /**
+   * Runs a command and returns its output and exit code in Object.
+   * Preserves the output when the execution fails.
+   * Stderr is redirected to stdout.
+   *
+   * @return {@link CommandReturn} object representation of stdout, stderr and exit code
+   */
+  public CommandReturn runWithOutput() throws IOException {
+    Process process = new ProcessBuilder(mCommand).redirectErrorStream(true).start();
+
+    try (BufferedReader inReader =
+                 new BufferedReader(new InputStreamReader(process.getInputStream()))) {
+      // read the output of the command
+      StringBuilder stdout = new StringBuilder();
+      String outLine = inReader.readLine();
+      while (outLine != null) {
+        stdout.append(outLine);
+        stdout.append("\n");
+        outLine = inReader.readLine();
+      }
+
+      // wait for the process to finish and check the exit code
+      int exitCode = process.waitFor();
+      if (exitCode != 0) {
+        // log error instead of throwing exception
+        LOG.warn(String.format("Non-zero exit code (%d) from command %s",
+                exitCode, Arrays.toString(mCommand)));
+      }
+
+      CommandReturn cr = new CommandReturn(exitCode, stdout.toString());
+
+      // destroy the process
+      if (process != null) {
+        process.destroy();
+      }
+
+      return cr;
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new IOException(e);
+    } finally {
+      if (process != null) {
+        process.destroy();
+      }
+    }
+  }
+}

core/common/src/main/java/alluxio/shell/SshCommand.java

@@ -0,0 +1,47 @@
+/*
+ * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0
+ * (the "License"). You may not use this work except in compliance with the License, which is
+ * available at www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied, as more fully set forth in the License.
+ *
+ * See the NOTICE file distributed with this work for information regarding copyright ownership.
+ */
+
+package alluxio.shell;
+
+import alluxio.util.ShellUtils;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * Object representation of a remote shell command by SSH.
+ */
+@NotThreadSafe
+public class SshCommand extends ShellCommand {
+  private final String mHostName;
+
+  /**
+   * Creates a SshCommand instance from the remote hostname and command.
+   *
+   * @param hostname the remote target hostname
+   * @param execString the command to execute
+   */
+  public SshCommand(String hostname, String[] execString) {
+    super(new String[]{"bash", "-c",
+            String.format("ssh %s %s %s", ShellUtils.COMMON_SSH_OPTS, hostname,
+                    String.join(" ", execString))});
+    mHostName = hostname;
+  }
+
+  /**
+   * Returns the remote target hostname.
+   *
+   * @return target hostname
+   */
+  public String getHostName() {
+    return mHostName;
+  }
+}
+