Skip to content

Commit

Permalink
Address comments. More fixes.
Browse files Browse the repository at this point in the history
  • Loading branch information
@maobaolong
maobaolong committed Jun 22, 2017
1 parent 0e33061 commit 42515f9
Show file tree
Hide file tree
Showing 4 changed files with 137 additions and 99 deletions.
53 changes: 24 additions & 29 deletions bin/alluxio
View file
Expand Up @@ -14,30 +14,30 @@ function printUsage {
echo "Usage: alluxio COMMAND [GENERIC_COMMAND_OPTIONS] [COMMAND_ARGS]"
echo
echo "COMMAND is one of:"
echo -e " format [-s] \t Format Alluxio master and all workers (if -s specified, only format if underfs is local and doesn't already exist)"
echo -e " formatWorker \t Format Alluxio worker storage"
echo -e " bootstrapConf \t Generate a config file if one doesn't exist"
echo -e " fs \t Command line tool for interacting with the Alluxio filesystem."
echo -e " getConf [key] \t Look up a configuration key, or print all configuration."
echo -e " loadufs \t Load existing files in underlayer filesystem into Alluxio."
echo -e " logLevel --logName=NAME [--target=<master,workers,host:port>] [--level=LEVEL]\t Setting and getting log level."
echo -e " runClass \t Run the main method of an Alluxio class."
echo -e " runTest \t Run an end-to-end test on an Alluxio cluster."
echo -e " runKVTest \t Run a test of key-value store operations."
echo -e " runTests \t Run all end-to-end tests on an Alluxio cluster."
echo -e " runJournalCrashTest \t Test the Master Journal System in a crash scenario. Try 'alluxio runJournalCrashTest -help' for more help."
echo -e " \t NOTE: This command will stop the existing server and creates a new one!"
echo -e " runMesosTest \t Test the Alluxio integration with Mesos. Try 'alluxio mesosTest -help' for more help."
echo -e " \t NOTE: This command requires Mesos to be running and will stop any Alluxio servers that are currently running."
echo -e " readJournal \t Read an Alluxio journal file from stdin and write a human-readable version of it to stdout."
echo -e " upgradeJournal \t Upgrade an Alluxio journal version 0 (Alluxio version < 1.5.0) to an Alluxio journal version 1 (Alluxio version >= 1.5.0)."
echo -e " killAll <WORD> \t Kill processes containing the WORD."
echo -e " copyDir <PATH> \t Copy the PATH to all worker nodes."
echo -e " clearCache \t Clear OS buffer cache of the machine."
echo -e " thriftGen \t Generate all thrift code."
echo -e " protoGen \t Generate all protocol buffer code."
echo -e " version \t Print Alluxio version and exit."
echo -e " validateConf \t Validate Alluxio conf and exit."
echo -e " format [-s] \t Format Alluxio master and all workers (if -s specified, only format if underfs is local and doesn't already exist)"
echo -e " formatWorker \t Format Alluxio worker storage"
echo -e " bootstrapConf \t Generate a config file if one doesn't exist"
echo -e " fs \t Command line tool for interacting with the Alluxio filesystem."
echo -e " getConf [key] \t Look up a configuration key, or print all configuration."
echo -e " loadufs \t Load existing files in underlayer filesystem into Alluxio."
echo -e " logLevel \t Set or get log level of Alluxio servers."
echo -e " runClass \t Run the main method of an Alluxio class."
echo -e " runTest \t Run an end-to-end test on an Alluxio cluster."
echo -e " runKVTest \t Run a test of key-value store operations."
echo -e " runTests \t Run all end-to-end tests on an Alluxio cluster."
echo -e " runJournalCrashTest\t Test the Master Journal System in a crash scenario. Try 'alluxio runJournalCrashTest -help' for more help."
echo -e " \t NOTE: This command will stop the existing server and creates a new one!"
echo -e " runMesosTest \t Test the Alluxio integration with Mesos. Try 'alluxio mesosTest -help' for more help."
echo -e " \t NOTE: This command requires Mesos to be running and will stop any Alluxio servers that are currently running."
echo -e " readJournal \t Read an Alluxio journal file from stdin and write a human-readable version of it to stdout."
echo -e " upgradeJournal\t Upgrade an Alluxio journal version 0 (Alluxio version < 1.5.0) to an Alluxio journal version 1 (Alluxio version >= 1.5.0)."
echo -e " killAll <WORD>\t Kill processes containing the WORD."
echo -e " copyDir <PATH>\t Copy the PATH to all worker nodes."
echo -e " clearCache \t Clear OS buffer cache of the machine."
echo -e " thriftGen \t Generate all thrift code."
echo -e " protoGen \t Generate all protocol buffer code."
echo -e " version \t Print Alluxio version and exit."
echo -e " validateConf \t Validate Alluxio conf and exit."
echo
echo "GENERIC_COMMAND_OPTIONS supports:"
echo -e " -D<property=value>\t Use a value for a given Alluxio property"
Expand Down Expand Up @@ -259,11 +259,6 @@ function main {
echo "Use the \"alluxio fs mount <AlluxioURI> <UfsURI>\" command followed by the \"alluxio fs ls -R <AlluxioURI>\" command instead." >&2
exit 1
;;
"logLevel")
CLASS="alluxio.cli.LogLevel"
CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH}
runJavaClass "$@"
;;
"runClass")
CLASS=$1
CLASSPATH=${ALLUXIO_CLIENT_CLASSPATH} # this should be the common case
Expand Down
22 changes: 8 additions & 14 deletions core/common/src/main/java/alluxio/util/network/HttpUtils.java
View file
Expand Up @@ -17,8 +17,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.InputStream;

/**
* Utility methods for working with http.
Expand All @@ -32,11 +32,10 @@ private HttpUtils() {
/**
* use post method to send params by http.
* @param url the http url
* @param timeout socket timeout and connection timeout
* @return the response body
* @param timeout millisecond to wait for the server to respond before giving up
* @return the response body stream
*/
public static String sendPost(String url, Integer timeout) {
StringBuilder contentBuffer = new StringBuilder();
public static InputStream post(String url, Integer timeout) throws IOException {
PostMethod postMethod = null;
try {
HttpClient httpClient = new HttpClient();
Expand All @@ -47,23 +46,18 @@ public static String sendPost(String url, Integer timeout) {
postMethod = new PostMethod(url);
int statusCode = httpClient.executeMethod(postMethod);
if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_CREATED) {
try (BufferedReader br = new BufferedReader(
new InputStreamReader(postMethod.getResponseBodyAsStream(), "UTF-8"))) {
String line;
while ((line = br.readLine()) != null) {
contentBuffer.append(line);
}
}
return postMethod.getResponseBodyAsStream();
} else {
LOG.error("HTTP POST error code:" + statusCode);
}
} catch (Exception e) {
LOG.error("HTTP POST error code:", e);
throw e;
} finally {
if (postMethod != null) {
postMethod.releaseConnection();
}
}
return contentBuffer.toString();
return null;
}
}
2 changes: 1 addition & 1 deletion core/common/src/main/java/alluxio/wire/LogInfo.java
View file
Expand Up @@ -73,6 +73,6 @@ public void setMessage(String message) {

@Override
public String toString() {
return mLogName + ":" + mLevel + "." + (mMessage != null ? " message: " + mMessage : "");
return mLevel + " " + mLogName + " - " + (mMessage != null ? mMessage : "");
}
}
159 changes: 104 additions & 55 deletions shell/src/main/java/alluxio/cli/LogLevel.java
View file
Expand Up @@ -21,7 +21,6 @@
import alluxio.wire.WorkerNetAddress;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
Expand All @@ -31,9 +30,11 @@
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang.StringUtils;
import org.apache.http.client.utils.URIBuilder;

import javax.annotation.concurrent.NotThreadSafe;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

Expand All @@ -47,22 +48,25 @@ public final class LogLevel {
private static final String TARGET_OPTION_NAME = "target";
private static final Option TATGET_OPTION =
Option.builder().required(false).longOpt(TARGET_OPTION_NAME).hasArg(true)
.desc("<master|workers|host:port>. Multi target split by ','.").build();
private static final String LOGNAME_OPTION_NAME = "logName";
private static final Option LOGNAME_OPTION =
Option.builder().required(true).longOpt(LOGNAME_OPTION_NAME).hasArg(true)
.desc("log name.").build();
.desc("<master|workers|host:port>."
+ " Multi target split by ',' host:port pair must be one of workers."
+ " Default target is master and all workers").build();
private static final String LOG_NAME_OPTION_NAME = "logName";
private static final Option LOG_NAME_OPTION =
Option.builder().required(true).longOpt(LOG_NAME_OPTION_NAME).hasArg(true)
.desc("The log name you want to get or set level.").build();
private static final String LEVEL_OPTION_NAME = "level";
private static final Option LEVEL_OPTION =
Option.builder().required(false).longOpt(LEVEL_OPTION_NAME).hasArg(true)
.desc("<master|workers|host:port>.").build();
.desc("The level of the log what you specified.").build();
private static final Options OPTIONS = new Options()
.addOption(TATGET_OPTION)
.addOption(LOGNAME_OPTION)
.addOption(LOG_NAME_OPTION)
.addOption(LEVEL_OPTION);

private static final String USAGE =
"logLevel --logName=LOGNAME [--target=<master|workers|host:port>] [--level=LEVEL]";
public static final String TARGET_SEPARATOR = ",";

/**
* Prints the help message.
Expand All @@ -72,13 +76,13 @@ public final class LogLevel {
public static void printHelp(String message) {
System.err.println(message);
HelpFormatter help = new HelpFormatter();
help.printHelp(USAGE, OPTIONS);
help.printHelp(LOG_LEVEL, OPTIONS, true);
}

/**
* Implements log level setting and getting.
*
* @param args list of arguments
* @param args list of arguments contains target, logName and level
* @return 0 on success, 1 on failures
*/
public static int logLevel(String... args) {
Expand All @@ -91,103 +95,148 @@ public static int logLevel(String... args) {
printHelp("Unable to parse input args: " + e.getMessage());
return 1;
}
Preconditions.checkNotNull(cmd, "Unable to parse input args");
List<TargetInfo> addrList = new ArrayList<>();
String logName = "";
String level = null;
String[] targets;
List<TargetInfo> targetInfoList = new ArrayList<>();

String argName = cmd.getOptionValue(LOGNAME_OPTION_NAME);
if (StringUtils.isNotBlank(argName)) {
logName = argName;
}
String[] targets = parseOptTarget(cmd);
String logName = parseOptLogName(cmd);
String level = parseOptLevel(cmd);

if (cmd.hasOption(TARGET_OPTION_NAME)) {
String argTarget = cmd.getOptionValue(TARGET_OPTION_NAME);
if (StringUtils.isBlank(argTarget)) {
targets = new String[]{"master", "workers"};
} else if (argTarget.contains(",")) {
targets = argTarget.split(",");
} else {
targets = new String[]{argTarget};
}
} else {
targets = new String[]{"master", "workers"};
}
for (String target : targets) {
if (target.contains("master")) {
String masterHost = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_WEB);
int masterPort = NetworkAddressUtils.getPort(ServiceType.MASTER_WEB);
addrList.add(new TargetInfo(masterHost + ":" + masterPort, "master"));
targetInfoList.add(new TargetInfo(masterHost, masterPort, "master"));
} else if (target.contains("workers")) {
AlluxioBlockStore alluxioBlockStore = AlluxioBlockStore.create();
try {
List<BlockWorkerInfo> workerInfoList = alluxioBlockStore.getWorkerInfoList();
for (BlockWorkerInfo workerInfo : workerInfoList) {
WorkerNetAddress netAddress = workerInfo.getNetAddress();
addrList.add(
new TargetInfo(netAddress.getHost() + ":" + netAddress.getWebPort(), "worker"));
targetInfoList.add(
new TargetInfo(netAddress.getHost(), netAddress.getWebPort(), "worker"));
}
} catch (IOException e) {
e.printStackTrace();
return 1;
}
} else if (target.contains(":")) {
addrList.add(new TargetInfo(target, "worker"));
String[] hostPortPair = target.split(":");
int port = Integer.parseInt(hostPortPair[1]);
targetInfoList.add(new TargetInfo(hostPortPair[0], port, "worker"));
}
}

for (TargetInfo targetInfo : targetInfoList) {
if (handleTarget(logName, level, targetInfo)) {
return 1;
}
}
return 0;
}

private static String[] parseOptTarget(CommandLine cmd) {
String[] targets;
if (cmd.hasOption(TARGET_OPTION_NAME)) {
String argTarget = cmd.getOptionValue(TARGET_OPTION_NAME);
if (StringUtils.isBlank(argTarget)) {
targets = new String[]{"master", "workers"};
} else if (argTarget.contains(TARGET_SEPARATOR)) {
targets = argTarget.split(TARGET_SEPARATOR);
} else {
targets = new String[]{argTarget};
}
} else {
targets = new String[]{"master", "workers"};
}
return targets;
}

private static String parseOptLogName(CommandLine cmd) {
String argName = cmd.getOptionValue(LOG_NAME_OPTION_NAME);
if (StringUtils.isNotBlank(argName)) {
return argName;
}
return "";
}

private static String parseOptLevel(CommandLine cmd) {
if (cmd.hasOption(LEVEL_OPTION_NAME)) {
String argLevel = cmd.getOptionValue(LEVEL_OPTION_NAME);
if (StringUtils.isNotBlank(argLevel)) {
level = argLevel;
return argLevel;
}
}
return null;
}

for (TargetInfo targetInfo : addrList) {
String url = "http://" + targetInfo.getAddres() + Constants.REST_API_PREFIX + "/"
+ targetInfo.getRole() + "/" + LOG_LEVEL + "?" + LOGNAME_OPTION_NAME + "=" + logName;
if (level != null) {
url += "&" + LEVEL_OPTION_NAME + "=" + level;
}
String result = HttpUtils.sendPost(url, 5000);
private static boolean handleTarget(String logName, String level, TargetInfo targetInfo) {
URIBuilder uriBuilder = new URIBuilder();
uriBuilder.setScheme("http");
uriBuilder.setHost(targetInfo.getHost());
uriBuilder.setPort(targetInfo.getPort());
uriBuilder.setPath(Constants.REST_API_PREFIX + "/" + targetInfo.getRole() + "/" + LOG_LEVEL);
uriBuilder.addParameter(LOG_NAME_OPTION_NAME, logName);
if (level != null) {
uriBuilder.addParameter(LEVEL_OPTION_NAME, level);
}
InputStream inputStream = null;
try {
inputStream = HttpUtils.post(uriBuilder.toString(), 5000);
} catch (IOException e) {
e.printStackTrace();
}
if (inputStream != null) {
ObjectMapper mapper = new ObjectMapper();
try {
LogInfo logInfo = mapper.readValue(result, LogInfo.class);
System.out.println(
targetInfo.getAddres() + "[" + targetInfo.getRole() + "]" + logInfo.toString());
LogInfo logInfo = mapper.readValue(inputStream, LogInfo.class);
System.out.println(targetInfo.toString() + logInfo.toString());
} catch (IOException e) {
e.printStackTrace();
return true;
}
} else {
return true;
}
return 0;
return false;
}

/**
* Prints Alluxio configuration.
* Set or get log level of Alluxio servers.
*
* @param args the arguments to specify the unit (optional) and configuration key (optional)
* @param args the arguments to specify the arguments of this command
*/
public static void main(String[] args) {
System.exit(logLevel(args));
}

private LogLevel() {} // this class is not intended for instantiation

static final class TargetInfo {
private String mAddres;
private static final class TargetInfo {
private String mRole;
private String mHost;
private int mPort;

public TargetInfo(String address, String role) {
mAddres = address;
public TargetInfo(String host, int port, String role) {
mHost = host;
mPort = port;
mRole = role;
}

public String getAddres() {
return mAddres;
public int getPort() {
return mPort;
}

public String getHost() {
return mHost;
}

public String getRole() {
return mRole;
}

@Override
public String toString() {
return mHost + ":" + mPort + "[" + mRole + "]";
}
}
}

0 comments on commit 42515f9

Please sign in to comment.