From 83a2ffec408fc3ebb89ab13ab4788d9f02fae33b Mon Sep 17 00:00:00 2001 From: "Vinod Kumar Vavilapalli (I am also known as @tshooter.)" Date: Mon, 8 Aug 2016 14:28:07 -0700 Subject: [PATCH] YARN-5470. Addedum to differentiate exactly matching of log-files with regex in yarn log CLI. Contributed by Xuan Gong. --- .../hadoop/yarn/client/cli/LogsCLI.java | 62 ++++++--- .../hadoop/yarn/client/cli/TestLogsCLI.java | 129 +++++++++++------- 2 files changed, 122 insertions(+), 69 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index ec045b2856e30..908d3795df6e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -84,6 +84,8 @@ public class LogsCLI extends Configured implements Tool { private static final String APP_OWNER_OPTION = "appOwner"; private static final String AM_CONTAINER_OPTION = "am"; private static final String PER_CONTAINER_LOG_FILES_OPTION = "log_files"; + private static final String PER_CONTAINER_LOG_FILES_REGEX_OPTION + = "log_files_pattern"; private static final String LIST_NODES_OPTION = "list_nodes"; private static final String SHOW_APPLICATION_LOG_INFO = "show_application_log_info"; @@ -91,7 +93,6 @@ public class LogsCLI extends Configured implements Tool { = "show_container_log_info"; private static final String OUT_OPTION = "out"; private static final String SIZE_OPTION = "size"; - private static final String REGEX_OPTION = "regex"; public static final String HELP_CMD = "help"; private PrintStream outStream = System.out; private YarnClient yarnClient = null; @@ -130,6 +131,7 @@ private int runCommand(String[] args) throws Exception { boolean showContainerLogInfo = false; boolean useRegex = false; String[] logFiles = null; + String[] logFilesRegex = null; List amContainersList = new ArrayList(); String localDir = null; long bytes = Long.MAX_VALUE; @@ -145,7 +147,6 @@ private int runCommand(String[] args) throws Exception { showApplicationLogInfo = commandLine.hasOption( SHOW_APPLICATION_LOG_INFO); showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO); - useRegex = commandLine.hasOption(REGEX_OPTION); if (getAMContainerLogs) { try { amContainersList = parseAMContainer(commandLine, printOpts); @@ -157,6 +158,11 @@ private int runCommand(String[] args) throws Exception { if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_OPTION)) { logFiles = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_OPTION); } + if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_REGEX_OPTION)) { + logFilesRegex = commandLine.getOptionValues( + PER_CONTAINER_LOG_FILES_REGEX_OPTION); + useRegex = true; + } if (commandLine.hasOption(SIZE_OPTION)) { bytes = Long.parseLong(commandLine.getOptionValue(SIZE_OPTION)); } @@ -206,6 +212,12 @@ private int runCommand(String[] args) throws Exception { return -1; } + if (logFiles != null && logFiles.length > 0 && logFilesRegex != null + && logFilesRegex.length > 0) { + System.err.println("Invalid options. Can only accept one of " + + "log_files/log_files_pattern."); + return -1; + } if (localDir != null) { File file = new File(localDir); if (file.exists() && file.isFile()) { @@ -248,10 +260,12 @@ private int runCommand(String[] args) throws Exception { } Set logs = new HashSet(); - if (fetchAllLogFiles(logFiles, useRegex)) { + if (fetchAllLogFiles(logFiles, logFilesRegex)) { logs.add("ALL"); } else if (logFiles != null && logFiles.length > 0) { logs.addAll(Arrays.asList(logFiles)); + } else if (logFilesRegex != null && logFilesRegex.length > 0) { + logs.addAll(Arrays.asList(logFilesRegex)); } ContainerLogsRequest request = new ContainerLogsRequest(appId, @@ -366,18 +380,28 @@ private List getAMContainerInfoForAHSWebService( return amContainersList; } - private boolean fetchAllLogFiles(String[] logFiles, boolean useRegex) { + private boolean fetchAllLogFiles(String[] logFiles, String[] logFilesRegex) { - // If no value is specified for the PER_CONTAINER_LOG_FILES_OPTION option, + // If no value is specified for the PER_CONTAINER_LOG_FILES_OPTION option + // and PER_CONTAINER_LOG_FILES_REGEX_OPTION // we will assume all logs. - if (logFiles == null || logFiles.length == 0) { + if ((logFiles == null || logFiles.length == 0) && ( + logFilesRegex == null || logFilesRegex.length == 0)) { return true; } - List logs = Arrays.asList(logFiles); - if (logs.contains("ALL") || logs.contains("*")|| - (logs.contains(".*") && useRegex)) { - return true; + if (logFiles != null && logFiles.length > 0) { + List logs = Arrays.asList(logFiles); + if (logs.contains("ALL") || logs.contains("*")) { + return true; + } + } + + if (logFilesRegex != null && logFilesRegex.length > 0) { + List logsRegex = Arrays.asList(logFilesRegex); + if (logsRegex.contains(".*")) { + return true; + } } return false; @@ -753,15 +777,20 @@ private Options createCommandOpts() { opts.addOption(amOption); Option logFileOpt = new Option(PER_CONTAINER_LOG_FILES_OPTION, true, "Specify comma-separated value " - + "to get exact matched log files. Use \"ALL\" or \"*\"to " - + "fetch all the log files for the container. Specific -regex " - + "for using java regex to find matched log files."); + + "to get exact matched log files. Use \"ALL\" or \"*\" to " + + "fetch all the log files for the container."); logFileOpt.setValueSeparator(','); logFileOpt.setArgs(Option.UNLIMITED_VALUES); logFileOpt.setArgName("Log File Name"); opts.addOption(logFileOpt); - opts.addOption(REGEX_OPTION, false, "Work with -log_files to find " - + "matched files by using java regex."); + Option logFileRegexOpt = new Option(PER_CONTAINER_LOG_FILES_REGEX_OPTION, + true, "Specify comma-separated value " + + "to get matched log files by using java regex. Use \".*\" to " + + "fetch all the log files for the container."); + logFileRegexOpt.setValueSeparator(','); + logFileRegexOpt.setArgs(Option.UNLIMITED_VALUES); + logFileRegexOpt.setArgName("Log File Pattern"); + opts.addOption(logFileRegexOpt); opts.addOption(SHOW_CONTAINER_LOG_INFO, false, "Show the container log metadata, " + "including log-file names, the size of the log files. " @@ -804,7 +833,8 @@ private Options createPrintOpts(Options commandOpts) { printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO)); printOpts.addOption(commandOpts.getOption(OUT_OPTION)); printOpts.addOption(commandOpts.getOption(SIZE_OPTION)); - printOpts.addOption(commandOpts.getOption(REGEX_OPTION)); + printOpts.addOption(commandOpts.getOption( + PER_CONTAINER_LOG_FILES_REGEX_OPTION)); return printOpts; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 467b2043ab512..19ddb2f63b028 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -189,57 +189,70 @@ public void testHelpMessage() throws Exception { pw.println("usage: yarn logs -applicationId [OPTIONS]"); pw.println(); pw.println("general options are:"); - pw.println(" -am Prints the AM Container logs for this"); - pw.println(" application. Specify comma-separated"); - pw.println(" value to get logs for related AM"); - pw.println(" Container. For example, If we specify -am"); - pw.println(" 1,2, we will get the logs for the first"); - pw.println(" AM Container as well as the second AM"); - pw.println(" Container. To get logs for all AM"); - pw.println(" Containers, use -am ALL. To get logs for"); - pw.println(" the latest AM Container, use -am -1. By"); - pw.println(" default, it will print all available"); - pw.println(" logs. Work with -log_files to get only"); - pw.println(" specific logs."); - pw.println(" -appOwner AppOwner (assumed to be current user if"); - pw.println(" not specified)"); - pw.println(" -containerId ContainerId. By default, it will print"); - pw.println(" all available logs. Work with -log_files"); - pw.println(" to get only specific logs. If specified,"); - pw.println(" the applicationId can be omitted"); - pw.println(" -help Displays help for all commands."); - pw.println(" -list_nodes Show the list of nodes that successfully"); - pw.println(" aggregated logs. This option can only be"); - pw.println(" used with finished applications."); - pw.println(" -log_files Specify comma-separated value to get"); - pw.println(" exact matched log files. Use \"ALL\" or"); - pw.println(" \"*\"to fetch all the log files for the"); - pw.println(" container. Specific -regex for using java"); - pw.println(" regex to find matched log files."); - pw.println(" -nodeAddress NodeAddress in the format nodename:port"); - pw.println(" -out Local directory for storing individual"); - pw.println(" container logs. The container logs will"); - pw.println(" be stored based on the node the container"); - pw.println(" ran on."); - pw.println(" -regex Work with -log_files to find matched"); - pw.println(" files by using java regex."); - pw.println(" -show_application_log_info Show the containerIds which belong to the"); - pw.println(" specific Application. You can combine"); - pw.println(" this with --nodeAddress to get"); - pw.println(" containerIds for all the containers on"); - pw.println(" the specific NodeManager."); - pw.println(" -show_container_log_info Show the container log metadata,"); - pw.println(" including log-file names, the size of the"); - pw.println(" log files. You can combine this with"); - pw.println(" --containerId to get log metadata for the"); - pw.println(" specific container, or with --nodeAddress"); - pw.println(" to get log metadata for all the"); - pw.println(" containers on the specific NodeManager."); - pw.println(" -size Prints the log file's first 'n' bytes or"); - pw.println(" the last 'n' bytes. Use negative values"); - pw.println(" as bytes to read from the end and"); - pw.println(" positive values as bytes to read from the"); - pw.println(" beginning."); + pw.println(" -am Prints the AM Container logs for"); + pw.println(" this application. Specify"); + pw.println(" comma-separated value to get logs"); + pw.println(" for related AM Container. For"); + pw.println(" example, If we specify -am 1,2,"); + pw.println(" we will get the logs for the"); + pw.println(" first AM Container as well as the"); + pw.println(" second AM Container. To get logs"); + pw.println(" for all AM Containers, use -am"); + pw.println(" ALL. To get logs for the latest"); + pw.println(" AM Container, use -am -1. By"); + pw.println(" default, it will print all"); + pw.println(" available logs. Work with"); + pw.println(" -log_files to get only specific"); + pw.println(" logs."); + pw.println(" -appOwner AppOwner (assumed to be current"); + pw.println(" user if not specified)"); + pw.println(" -containerId ContainerId. By default, it will"); + pw.println(" print all available logs. Work"); + pw.println(" with -log_files to get only"); + pw.println(" specific logs. If specified, the"); + pw.println(" applicationId can be omitted"); + pw.println(" -help Displays help for all commands."); + pw.println(" -list_nodes Show the list of nodes that"); + pw.println(" successfully aggregated logs."); + pw.println(" This option can only be used with"); + pw.println(" finished applications."); + pw.println(" -log_files Specify comma-separated value to"); + pw.println(" get exact matched log files. Use"); + pw.println(" \"ALL\" or \"*\" to fetch all the log"); + pw.println(" files for the container."); + pw.println(" -log_files_pattern Specify comma-separated value to"); + pw.println(" get matched log files by using"); + pw.println(" java regex. Use \".*\" to fetch all"); + pw.println(" the log files for the container."); + pw.println(" -nodeAddress NodeAddress in the format"); + pw.println(" nodename:port"); + pw.println(" -out Local directory for storing"); + pw.println(" individual container logs. The"); + pw.println(" container logs will be stored"); + pw.println(" based on the node the container"); + pw.println(" ran on."); + pw.println(" -show_application_log_info Show the containerIds which"); + pw.println(" belong to the specific"); + pw.println(" Application. You can combine this"); + pw.println(" with --nodeAddress to get"); + pw.println(" containerIds for all the"); + pw.println(" containers on the specific"); + pw.println(" NodeManager."); + pw.println(" -show_container_log_info Show the container log metadata,"); + pw.println(" including log-file names, the"); + pw.println(" size of the log files. You can"); + pw.println(" combine this with --containerId"); + pw.println(" to get log metadata for the"); + pw.println(" specific container, or with"); + pw.println(" --nodeAddress to get log metadata"); + pw.println(" for all the containers on the"); + pw.println(" specific NodeManager."); + pw.println(" -size Prints the log file's first 'n'"); + pw.println(" bytes or the last 'n' bytes. Use"); + pw.println(" negative values as bytes to read"); + pw.println(" from the end and positive values"); + pw.println(" as bytes to read from the"); + pw.println(" beginning."); pw.close(); String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); @@ -346,7 +359,7 @@ public ContainerReport getContainerReport(String containerIdStr) sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), - "-log_files", ".*", "-regex"}); + "-log_files_pattern", ".*"}); assertTrue(exitCode == 0); assertTrue(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); @@ -392,7 +405,7 @@ public ContainerReport getContainerReport(String containerIdStr) sysOutStream.reset(); exitCode = cli.run(new String[] {"-applicationId", appId.toString(), - "-log_files", "std*", "-regex"}); + "-log_files_pattern", "std*"}); assertTrue(exitCode == 0); assertFalse(sysOutStream.toString().contains( logMessage(containerId1, "syslog"))); @@ -809,6 +822,16 @@ public void testLogsCLIWithInvalidArgs() throws Exception { + "show_container_log_info.")); sysErrStream.reset(); + // Specify log_files and log_files_pattern + // at the same time + exitCode = cli.run(new String[] {"-applicationId", appId.toString(), + "-log_files", "*", "-log_files_pattern", ".*"}); + assertTrue(exitCode == -1); + assertTrue(sysErrStream.toString().contains("Invalid options. " + + "Can only accept one of log_files/" + + "log_files_pattern.")); + sysErrStream.reset(); + // Specify a file name to the option -out try { fs.mkdirs(localPath);