From 42515f9977853c47eb1efac79f34f77edefd462f Mon Sep 17 00:00:00 2001 From: maobaolong Date: Thu, 22 Jun 2017 21:52:35 +0800 Subject: [PATCH] Address comments. More fixes. --- bin/alluxio | 53 +++--- .../java/alluxio/util/network/HttpUtils.java | 22 +-- .../src/main/java/alluxio/wire/LogInfo.java | 2 +- shell/src/main/java/alluxio/cli/LogLevel.java | 159 ++++++++++++------ 4 files changed, 137 insertions(+), 99 deletions(-) diff --git a/bin/alluxio b/bin/alluxio index 443ae8e627b9..a836322aa66b 100755 --- a/bin/alluxio +++ b/bin/alluxio @@ -14,30 +14,30 @@ function printUsage { echo "Usage: alluxio COMMAND [GENERIC_COMMAND_OPTIONS] [COMMAND_ARGS]" echo echo "COMMAND is one of:" - echo -e " format [-s] \t Format Alluxio master and all workers (if -s specified, only format if underfs is local and doesn't already exist)" - echo -e " formatWorker \t Format Alluxio worker storage" - echo -e " bootstrapConf \t Generate a config file if one doesn't exist" - echo -e " fs \t Command line tool for interacting with the Alluxio filesystem." - echo -e " getConf [key] \t Look up a configuration key, or print all configuration." - echo -e " loadufs \t Load existing files in underlayer filesystem into Alluxio." - echo -e " logLevel --logName=NAME [--target=] [--level=LEVEL]\t Setting and getting log level." - echo -e " runClass \t Run the main method of an Alluxio class." - echo -e " runTest \t Run an end-to-end test on an Alluxio cluster." - echo -e " runKVTest \t Run a test of key-value store operations." - echo -e " runTests \t Run all end-to-end tests on an Alluxio cluster." - echo -e " runJournalCrashTest \t Test the Master Journal System in a crash scenario. Try 'alluxio runJournalCrashTest -help' for more help." - echo -e " \t NOTE: This command will stop the existing server and creates a new one!" - echo -e " runMesosTest \t Test the Alluxio integration with Mesos. Try 'alluxio mesosTest -help' for more help." - echo -e " \t NOTE: This command requires Mesos to be running and will stop any Alluxio servers that are currently running." - echo -e " readJournal \t Read an Alluxio journal file from stdin and write a human-readable version of it to stdout." - echo -e " upgradeJournal \t Upgrade an Alluxio journal version 0 (Alluxio version < 1.5.0) to an Alluxio journal version 1 (Alluxio version >= 1.5.0)." - echo -e " killAll \t Kill processes containing the WORD." - echo -e " copyDir \t Copy the PATH to all worker nodes." - echo -e " clearCache \t Clear OS buffer cache of the machine." - echo -e " thriftGen \t Generate all thrift code." - echo -e " protoGen \t Generate all protocol buffer code." - echo -e " version \t Print Alluxio version and exit." - echo -e " validateConf \t Validate Alluxio conf and exit." + echo -e " format [-s] \t Format Alluxio master and all workers (if -s specified, only format if underfs is local and doesn't already exist)" + echo -e " formatWorker \t Format Alluxio worker storage" + echo -e " bootstrapConf \t Generate a config file if one doesn't exist" + echo -e " fs \t Command line tool for interacting with the Alluxio filesystem." + echo -e " getConf [key] \t Look up a configuration key, or print all configuration." + echo -e " loadufs \t Load existing files in underlayer filesystem into Alluxio." + echo -e " logLevel \t Set or get log level of Alluxio servers." + echo -e " runClass \t Run the main method of an Alluxio class." + echo -e " runTest \t Run an end-to-end test on an Alluxio cluster." + echo -e " runKVTest \t Run a test of key-value store operations." + echo -e " runTests \t Run all end-to-end tests on an Alluxio cluster." + echo -e " runJournalCrashTest\t Test the Master Journal System in a crash scenario. Try 'alluxio runJournalCrashTest -help' for more help." + echo -e " \t NOTE: This command will stop the existing server and creates a new one!" + echo -e " runMesosTest \t Test the Alluxio integration with Mesos. Try 'alluxio mesosTest -help' for more help." + echo -e " \t NOTE: This command requires Mesos to be running and will stop any Alluxio servers that are currently running." + echo -e " readJournal \t Read an Alluxio journal file from stdin and write a human-readable version of it to stdout." + echo -e " upgradeJournal\t Upgrade an Alluxio journal version 0 (Alluxio version < 1.5.0) to an Alluxio journal version 1 (Alluxio version >= 1.5.0)." + echo -e " killAll \t Kill processes containing the WORD." + echo -e " copyDir \t Copy the PATH to all worker nodes." + echo -e " clearCache \t Clear OS buffer cache of the machine." + echo -e " thriftGen \t Generate all thrift code." + echo -e " protoGen \t Generate all protocol buffer code." + echo -e " version \t Print Alluxio version and exit." + echo -e " validateConf \t Validate Alluxio conf and exit." echo echo "GENERIC_COMMAND_OPTIONS supports:" echo -e " -D\t Use a value for a given Alluxio property" @@ -259,11 +259,6 @@ function main { echo "Use the \"alluxio fs mount \" command followed by the \"alluxio fs ls -R \" command instead." >&2 exit 1 ;; - "logLevel") - CLASS="alluxio.cli.LogLevel" - CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} - runJavaClass "$@" - ;; "runClass") CLASS=$1 CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} # this should be the common case diff --git a/core/common/src/main/java/alluxio/util/network/HttpUtils.java b/core/common/src/main/java/alluxio/util/network/HttpUtils.java index e68dd1b3ea74..9aa028968ee2 100644 --- a/core/common/src/main/java/alluxio/util/network/HttpUtils.java +++ b/core/common/src/main/java/alluxio/util/network/HttpUtils.java @@ -17,8 +17,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.InputStreamReader; +import java.io.IOException; +import java.io.InputStream; /** * Utility methods for working with http. @@ -32,11 +32,10 @@ private HttpUtils() { /** * use post method to send params by http. * @param url the http url - * @param timeout socket timeout and connection timeout - * @return the response body + * @param timeout millisecond to wait for the server to respond before giving up + * @return the response body stream */ - public static String sendPost(String url, Integer timeout) { - StringBuilder contentBuffer = new StringBuilder(); + public static InputStream post(String url, Integer timeout) throws IOException { PostMethod postMethod = null; try { HttpClient httpClient = new HttpClient(); @@ -47,23 +46,18 @@ public static String sendPost(String url, Integer timeout) { postMethod = new PostMethod(url); int statusCode = httpClient.executeMethod(postMethod); if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_CREATED) { - try (BufferedReader br = new BufferedReader( - new InputStreamReader(postMethod.getResponseBodyAsStream(), "UTF-8"))) { - String line; - while ((line = br.readLine()) != null) { - contentBuffer.append(line); - } - } + return postMethod.getResponseBodyAsStream(); } else { LOG.error("HTTP POST error code:" + statusCode); } } catch (Exception e) { LOG.error("HTTP POST error code:", e); + throw e; } finally { if (postMethod != null) { postMethod.releaseConnection(); } } - return contentBuffer.toString(); + return null; } } diff --git a/core/common/src/main/java/alluxio/wire/LogInfo.java b/core/common/src/main/java/alluxio/wire/LogInfo.java index 30431f8a748b..c16615ea1ced 100644 --- a/core/common/src/main/java/alluxio/wire/LogInfo.java +++ b/core/common/src/main/java/alluxio/wire/LogInfo.java @@ -73,6 +73,6 @@ public void setMessage(String message) { @Override public String toString() { - return mLogName + ":" + mLevel + "." + (mMessage != null ? " message: " + mMessage : ""); + return mLevel + " " + mLogName + " - " + (mMessage != null ? mMessage : ""); } } diff --git a/shell/src/main/java/alluxio/cli/LogLevel.java b/shell/src/main/java/alluxio/cli/LogLevel.java index 1b4a77958cc1..6ae0cd4c9e1f 100644 --- a/shell/src/main/java/alluxio/cli/LogLevel.java +++ b/shell/src/main/java/alluxio/cli/LogLevel.java @@ -21,7 +21,6 @@ import alluxio.wire.WorkerNetAddress; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; @@ -31,9 +30,11 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.lang.StringUtils; +import org.apache.http.client.utils.URIBuilder; import javax.annotation.concurrent.NotThreadSafe; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.List; @@ -47,22 +48,25 @@ public final class LogLevel { private static final String TARGET_OPTION_NAME = "target"; private static final Option TATGET_OPTION = Option.builder().required(false).longOpt(TARGET_OPTION_NAME).hasArg(true) - .desc(". Multi target split by ','.").build(); - private static final String LOGNAME_OPTION_NAME = "logName"; - private static final Option LOGNAME_OPTION = - Option.builder().required(true).longOpt(LOGNAME_OPTION_NAME).hasArg(true) - .desc("log name.").build(); + .desc("." + + " Multi target split by ',' host:port pair must be one of workers." + + " Default target is master and all workers").build(); + private static final String LOG_NAME_OPTION_NAME = "logName"; + private static final Option LOG_NAME_OPTION = + Option.builder().required(true).longOpt(LOG_NAME_OPTION_NAME).hasArg(true) + .desc("The log name you want to get or set level.").build(); private static final String LEVEL_OPTION_NAME = "level"; private static final Option LEVEL_OPTION = Option.builder().required(false).longOpt(LEVEL_OPTION_NAME).hasArg(true) - .desc(".").build(); + .desc("The level of the log what you specified.").build(); private static final Options OPTIONS = new Options() .addOption(TATGET_OPTION) - .addOption(LOGNAME_OPTION) + .addOption(LOG_NAME_OPTION) .addOption(LEVEL_OPTION); private static final String USAGE = "logLevel --logName=LOGNAME [--target=] [--level=LEVEL]"; + public static final String TARGET_SEPARATOR = ","; /** * Prints the help message. @@ -72,13 +76,13 @@ public final class LogLevel { public static void printHelp(String message) { System.err.println(message); HelpFormatter help = new HelpFormatter(); - help.printHelp(USAGE, OPTIONS); + help.printHelp(LOG_LEVEL, OPTIONS, true); } /** * Implements log level setting and getting. * - * @param args list of arguments + * @param args list of arguments contains target, logName and level * @return 0 on success, 1 on failures */ public static int logLevel(String... args) { @@ -91,81 +95,115 @@ public static int logLevel(String... args) { printHelp("Unable to parse input args: " + e.getMessage()); return 1; } - Preconditions.checkNotNull(cmd, "Unable to parse input args"); - List addrList = new ArrayList<>(); - String logName = ""; - String level = null; - String[] targets; + List targetInfoList = new ArrayList<>(); - String argName = cmd.getOptionValue(LOGNAME_OPTION_NAME); - if (StringUtils.isNotBlank(argName)) { - logName = argName; - } + String[] targets = parseOptTarget(cmd); + String logName = parseOptLogName(cmd); + String level = parseOptLevel(cmd); - if (cmd.hasOption(TARGET_OPTION_NAME)) { - String argTarget = cmd.getOptionValue(TARGET_OPTION_NAME); - if (StringUtils.isBlank(argTarget)) { - targets = new String[]{"master", "workers"}; - } else if (argTarget.contains(",")) { - targets = argTarget.split(","); - } else { - targets = new String[]{argTarget}; - } - } else { - targets = new String[]{"master", "workers"}; - } for (String target : targets) { if (target.contains("master")) { String masterHost = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_WEB); int masterPort = NetworkAddressUtils.getPort(ServiceType.MASTER_WEB); - addrList.add(new TargetInfo(masterHost + ":" + masterPort, "master")); + targetInfoList.add(new TargetInfo(masterHost, masterPort, "master")); } else if (target.contains("workers")) { AlluxioBlockStore alluxioBlockStore = AlluxioBlockStore.create(); try { List workerInfoList = alluxioBlockStore.getWorkerInfoList(); for (BlockWorkerInfo workerInfo : workerInfoList) { WorkerNetAddress netAddress = workerInfo.getNetAddress(); - addrList.add( - new TargetInfo(netAddress.getHost() + ":" + netAddress.getWebPort(), "worker")); + targetInfoList.add( + new TargetInfo(netAddress.getHost(), netAddress.getWebPort(), "worker")); } } catch (IOException e) { e.printStackTrace(); + return 1; } } else if (target.contains(":")) { - addrList.add(new TargetInfo(target, "worker")); + String[] hostPortPair = target.split(":"); + int port = Integer.parseInt(hostPortPair[1]); + targetInfoList.add(new TargetInfo(hostPortPair[0], port, "worker")); + } + } + + for (TargetInfo targetInfo : targetInfoList) { + if (handleTarget(logName, level, targetInfo)) { + return 1; + } + } + return 0; + } + + private static String[] parseOptTarget(CommandLine cmd) { + String[] targets; + if (cmd.hasOption(TARGET_OPTION_NAME)) { + String argTarget = cmd.getOptionValue(TARGET_OPTION_NAME); + if (StringUtils.isBlank(argTarget)) { + targets = new String[]{"master", "workers"}; + } else if (argTarget.contains(TARGET_SEPARATOR)) { + targets = argTarget.split(TARGET_SEPARATOR); + } else { + targets = new String[]{argTarget}; } + } else { + targets = new String[]{"master", "workers"}; + } + return targets; + } + + private static String parseOptLogName(CommandLine cmd) { + String argName = cmd.getOptionValue(LOG_NAME_OPTION_NAME); + if (StringUtils.isNotBlank(argName)) { + return argName; } + return ""; + } + private static String parseOptLevel(CommandLine cmd) { if (cmd.hasOption(LEVEL_OPTION_NAME)) { String argLevel = cmd.getOptionValue(LEVEL_OPTION_NAME); if (StringUtils.isNotBlank(argLevel)) { - level = argLevel; + return argLevel; } } + return null; + } - for (TargetInfo targetInfo : addrList) { - String url = "http://" + targetInfo.getAddres() + Constants.REST_API_PREFIX + "/" - + targetInfo.getRole() + "/" + LOG_LEVEL + "?" + LOGNAME_OPTION_NAME + "=" + logName; - if (level != null) { - url += "&" + LEVEL_OPTION_NAME + "=" + level; - } - String result = HttpUtils.sendPost(url, 5000); + private static boolean handleTarget(String logName, String level, TargetInfo targetInfo) { + URIBuilder uriBuilder = new URIBuilder(); + uriBuilder.setScheme("http"); + uriBuilder.setHost(targetInfo.getHost()); + uriBuilder.setPort(targetInfo.getPort()); + uriBuilder.setPath(Constants.REST_API_PREFIX + "/" + targetInfo.getRole() + "/" + LOG_LEVEL); + uriBuilder.addParameter(LOG_NAME_OPTION_NAME, logName); + if (level != null) { + uriBuilder.addParameter(LEVEL_OPTION_NAME, level); + } + InputStream inputStream = null; + try { + inputStream = HttpUtils.post(uriBuilder.toString(), 5000); + } catch (IOException e) { + e.printStackTrace(); + } + if (inputStream != null) { ObjectMapper mapper = new ObjectMapper(); try { - LogInfo logInfo = mapper.readValue(result, LogInfo.class); - System.out.println( - targetInfo.getAddres() + "[" + targetInfo.getRole() + "]" + logInfo.toString()); + LogInfo logInfo = mapper.readValue(inputStream, LogInfo.class); + System.out.println(targetInfo.toString() + logInfo.toString()); } catch (IOException e) { e.printStackTrace(); + return true; } + } else { + return true; } - return 0; + return false; } /** - * Prints Alluxio configuration. + * Set or get log level of Alluxio servers. * - * @param args the arguments to specify the unit (optional) and configuration key (optional) + * @param args the arguments to specify the arguments of this command */ public static void main(String[] args) { System.exit(logLevel(args)); @@ -173,21 +211,32 @@ public static void main(String[] args) { private LogLevel() {} // this class is not intended for instantiation - static final class TargetInfo { - private String mAddres; + private static final class TargetInfo { private String mRole; + private String mHost; + private int mPort; - public TargetInfo(String address, String role) { - mAddres = address; + public TargetInfo(String host, int port, String role) { + mHost = host; + mPort = port; mRole = role; } - public String getAddres() { - return mAddres; + public int getPort() { + return mPort; + } + + public String getHost() { + return mHost; } public String getRole() { return mRole; } + + @Override + public String toString() { + return mHost + ":" + mPort + "[" + mRole + "]"; + } } }