Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

CLOSES OOZIE:7 Log retrieval for large number of actions particular c…

…oorinator actions
  • Loading branch information...
commit 9591abbac64c456d5724d51eb876d8c77609a822 1 parent 575d71c
@kirann kirann authored Mohammad Kamrul Islam committed
View
34 client/src/main/java/org/apache/oozie/cli/OozieCLI.java
@@ -19,6 +19,7 @@
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.PrintStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
@@ -94,6 +95,7 @@
public static final String RERUN_OPTION = "rerun";
public static final String INFO_OPTION = "info";
public static final String LOG_OPTION = "log";
+ public static final String LOG_ACTION_OPTION = "action";
public static final String DEFINITION_OPTION = "definition";
public static final String CONFIG_CONTENT_OPTION = "configcontent";
@@ -199,6 +201,8 @@ protected Options createJobOptions() {
Option len = new Option(LEN_OPTION, true, "number of actions (default TOTAL ACTIONS, requires -info)");
Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (default GMT)");
Option log = new Option(LOG_OPTION, true, "job log");
+ Option log_action = new Option(LOG_ACTION_OPTION, true,
+ "coordinator log retrieval on action ids (requires -log)");
Option definition = new Option(DEFINITION_OPTION, true, "job definition");
Option config_content = new Option(CONFIG_CONTENT_OPTION, true, "job configuration");
Option verbose = new Option(VERBOSE_OPTION, false, "verbose mode");
@@ -242,6 +246,7 @@ protected Options createJobOptions() {
jobOptions.addOption(rerun_coord);
jobOptions.addOption(rerun_refresh);
jobOptions.addOption(rerun_nocleanup);
+ jobOptions.addOption(log_action);
jobOptions.addOptionGroup(actions);
return jobOptions;
}
@@ -695,7 +700,30 @@ else if (commandLine.getOptionValue(INFO_OPTION).contains("-W@")) {
}
}
else if (options.contains(LOG_OPTION)) {
- System.out.println(wc.getJobLog(commandLine.getOptionValue(LOG_OPTION)));
+ PrintStream ps = System.out;
+ if (commandLine.getOptionValue(LOG_OPTION).contains("-C")) {
+ String logRetrievalScope = null;
+ String logRetrievalType = null;
+ if (options.contains(LOG_ACTION_OPTION)) {
+ logRetrievalType = RestConstants.JOB_LOG_ACTION;
+ logRetrievalScope = commandLine.getOptionValue(LOG_ACTION_OPTION);
+ }
+ try {
+ wc.getJobLog(commandLine.getOptionValue(LOG_OPTION), logRetrievalType, logRetrievalScope, ps);
+ }
+ finally {
+ ps.close();
+ }
+ }
+ else {
+ if (!options.contains(LOG_ACTION_OPTION)) {
+ wc.getJobLog(commandLine.getOptionValue(LOG_OPTION), null, null, ps);
+ }
+ else {
+ throw new OozieCLIException("Invalid options provided for log retrieval. " + LOG_ACTION_OPTION
+ + " is valid only for coordinator job log retrieval");
+ }
+ }
}
else if (options.contains(DEFINITION_OPTION)) {
System.out.println(wc.getJobDefinition(commandLine.getOptionValue(DEFINITION_OPTION)));
@@ -1035,8 +1063,8 @@ private void printBundleJobs(List<BundleJob> jobs, boolean localtime, boolean ve
for (BundleJob job : jobs) {
System.out.println(String.format(BUNDLE_JOBS_FORMATTER, maskIfNull(job.getId()), maskIfNull(job
- .getAppName()), job.getStatus(), maskDate(job.getKickoffTime(), localtime),
- maskDate(job.getCreatedTime(), localtime), maskIfNull(job.getUser()), maskIfNull(job.getGroup())));
+ .getAppName()), job.getStatus(), maskDate(job.getKickoffTime(), localtime), maskDate(job
+ .getCreatedTime(), localtime), maskIfNull(job.getUser()), maskIfNull(job.getGroup())));
System.out.println(RULER);
}
}
View
86 client/src/main/java/org/apache/oozie/client/OozieClient.java
@@ -16,8 +16,10 @@
import java.io.BufferedReader;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
+import java.io.PrintStream;
import java.io.Reader;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -125,7 +127,7 @@
public static enum SYSTEM_MODE {
NORMAL, NOWEBSERVICE, SAFEMODE
};
-
+
/**
* debugMode =0 means no debugging. > 0 means debugging on.
*/
@@ -372,8 +374,8 @@ public T call() throws OozieClientException {
return call(conn);
}
else {
- System.out
- .println("Option not supported in target server. Supported only on Oozie-2.0 or greater. Use 'oozie help' for details");
+ System.out.println("Option not supported in target server. Supported only on Oozie-2.0 or greater."
+ + " Use 'oozie help' for details");
throw new OozieClientException(OozieClientException.UNSUPPORTED_VERSION, new Exception());
}
}
@@ -687,11 +689,28 @@ public String getJobLog(String jobId) throws OozieClientException {
return new JobLog(jobId).call();
}
- private class JobLog extends JobMetadata {
+ /**
+ * Get the log of a job.
+ *
+ * @param jobId job Id.
+ * @param logRetrievalType Based on which filter criteria the log is retrieved
+ * @param logRetrievalScope Value for the retrieval type
+ * @param ps Printstream of command line interface
+ * @throws OozieClientException thrown if the job info could not be retrieved.
+ */
+ public void getJobLog(String jobId, String logRetrievalType, String logRetrievalScope, PrintStream ps)
+ throws OozieClientException {
+ new JobLog(jobId, logRetrievalType, logRetrievalScope, ps).call();
+ }
+ private class JobLog extends JobMetadata {
JobLog(String jobId) {
super(jobId, RestConstants.JOB_SHOW_LOG);
}
+
+ JobLog(String jobId, String logRetrievalType, String logRetrievalScope, PrintStream ps) {
+ super(jobId, logRetrievalType, logRetrievalScope, RestConstants.JOB_SHOW_LOG, ps);
+ }
}
/**
@@ -713,23 +732,73 @@ public String getJobDefinition(String jobId) throws OozieClientException {
}
private class JobMetadata extends ClientCallable<String> {
+ PrintStream printStream;
JobMetadata(String jobId, String metaType) {
super("GET", RestConstants.JOB, notEmpty(jobId, "jobId"), prepareParams(RestConstants.JOB_SHOW_PARAM,
metaType));
}
+ JobMetadata(String jobId, String logRetrievalType, String logRetrievalScope, String metaType, PrintStream ps) {
+ super("GET", RestConstants.JOB, notEmpty(jobId, "jobId"), prepareParams(RestConstants.JOB_SHOW_PARAM,
+ metaType, RestConstants.JOB_LOG_TYPE_PARAM, logRetrievalType, RestConstants.JOB_LOG_SCOPE_PARAM,
+ logRetrievalScope));
+ printStream = ps;
+ }
+
@Override
protected String call(HttpURLConnection conn) throws IOException, OozieClientException {
+ String returnVal = null;
if ((conn.getResponseCode() == HttpURLConnection.HTTP_OK)) {
-
- String output = getReaderAsString(new InputStreamReader(conn.getInputStream()), -1);
- return output;
+ InputStream is = conn.getInputStream();
+ InputStreamReader isr = new InputStreamReader(is);
+ try {
+ if (printStream != null) {
+ sendToOutputStream(isr, -1);
+ }
+ else {
+ returnVal = getReaderAsString(isr, -1);
+ }
+ }
+ finally {
+ isr.close();
+ }
}
else {
handleError(conn);
}
- return null;
+ return returnVal;
+ }
+
+ /**
+ * Output the log to command line interface
+ *
+ * @param reader reader to read into a string.
+ * @param maxLen max content length allowed, if -1 there is no limit.
+ * @param ps Printstream of command line interface
+ * @throws IOException
+ */
+ private void sendToOutputStream(Reader reader, int maxLen) throws IOException {
+ if (reader == null) {
+ throw new IllegalArgumentException("reader cannot be null");
+ }
+ StringBuilder sb = new StringBuilder();
+ char[] buffer = new char[2048];
+ int read;
+ int count = 0;
+ int noOfCharstoFlush = 1024;
+ while ((read = reader.read(buffer)) > -1) {
+ count += read;
+ if ((maxLen > -1) && (count > maxLen)) {
+ break;
+ }
+ sb.append(buffer, 0, read);
+ if (sb.length() > noOfCharstoFlush) {
+ printStream.print(sb.toString());
+ sb = new StringBuilder("");
+ }
+ }
+ printStream.print(sb.toString());
}
/**
@@ -745,7 +814,6 @@ private String getReaderAsString(Reader reader, int maxLen) throws IOException {
if (reader == null) {
throw new IllegalArgumentException("reader cannot be null");
}
-
StringBuffer sb = new StringBuffer();
char[] buffer = new char[2048];
int read;
View
12 client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
@@ -59,7 +59,7 @@
public static final String JOB_ACTION_RERUN = "rerun";
public static final String JOB_COORD_ACTION_RERUN = "coord-rerun";
-
+
public static final String JOB_BUNDLE_ACTION_RERUN = "bundle-rerun";
public static final String JOB_SHOW_PARAM = "show";
@@ -73,9 +73,9 @@
public static final String JOB_SHOW_DEFINITION = "definition";
public static final String JOB_BUNDLE_RERUN_COORD_SCOPE_PARAM = "coord-scope";
-
+
public static final String JOB_BUNDLE_RERUN_DATE_SCOPE_PARAM = "date-scope";
-
+
public static final String JOB_COORD_RERUN_TYPE_PARAM = "type";
public static final String JOB_COORD_RERUN_DATE = "date";
@@ -88,6 +88,12 @@
public static final String JOB_COORD_RERUN_NOCLEANUP_PARAM = "nocleanup";
+ public static final String JOB_LOG_ACTION = "action";
+
+ public static final String JOB_LOG_SCOPE_PARAM = "scope";
+
+ public static final String JOB_LOG_TYPE_PARAM = "type";
+
public static final String JOBS_FILTER_PARAM = "filter";
public static final String JOBS_EXTERNAL_ID_PARAM = "external-id";
View
16 core/src/main/java/org/apache/oozie/BaseEngine.java
@@ -51,7 +51,7 @@
public abstract class BaseEngine {
public static final String USE_XCOMMAND = "oozie.useXCommand";
-
+
protected String user;
protected String authToken;
@@ -74,7 +74,9 @@ protected String getAuthToken() {
}
/**
- * Submit a job. <p/> It validates configuration properties.
+ * Submit a job.
+ * <p/>
+ * It validates configuration properties.
*
* @param conf job configuration.
* @param startJob indicates if the job should be started or not.
@@ -133,7 +135,6 @@ protected String getAuthToken() {
*/
public abstract void reRun(String jobId, Configuration conf) throws BaseEngineException;
-
/**
* Return the info about a wf job.
*
@@ -190,12 +191,14 @@ protected String getAuthToken() {
* @param writer writer to stream the log to.
* @throws IOException thrown if the log cannot be streamed.
* @throws BaseEngineException thrown if there is error in getting the Workflow/Coordinator Job Information for
- * jobId.
+ * jobId.
*/
public abstract void streamLog(String jobId, Writer writer) throws IOException, BaseEngineException;
/**
- * Return the workflow Job ID for an external ID. <p/> This is reverse lookup for recovery purposes.
+ * Return the workflow Job ID for an external ID.
+ * <p/>
+ * This is reverse lookup for recovery purposes.
*
* @param externalId external ID provided at job submission time.
* @return the associated workflow job ID if any, <code>null</code> if none.
@@ -203,7 +206,6 @@ protected String getAuthToken() {
*/
public abstract String getJobIdForExternalId(String externalId) throws BaseEngineException;
- public abstract String dryrunSubmit(Configuration conf, boolean startJob)
- throws BaseEngineException;
+ public abstract String dryrunSubmit(Configuration conf, boolean startJob) throws BaseEngineException;
}
View
89 core/src/main/java/org/apache/oozie/CoordinatorEngine.java
@@ -20,15 +20,16 @@
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.client.CoordinatorJob;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.WorkflowJob;
+import org.apache.oozie.client.rest.RestConstants;
import org.apache.oozie.command.CommandException;
import org.apache.oozie.command.coord.CoordActionInfoCommand;
import org.apache.oozie.command.coord.CoordActionInfoXCommand;
@@ -242,7 +243,7 @@ public void reRun(String jobId, Configuration conf) throws BaseEngineException {
public CoordinatorActionInfo reRun(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup)
throws BaseEngineException {
try {
- if (useXCommand) {
+ if (useXCommand) {
return new CoordRerunXCommand(jobId, rerunType, scope, refresh, noCleanup).call();
}
else {
@@ -295,6 +296,84 @@ public void streamLog(String jobId, Writer writer) throws IOException, BaseEngin
Services.get().get(XLogService.class).streamLog(filter, job.getCreatedTime(), new Date(), writer);
}
+ /**
+ * Add list of actions to the filter based on conditions
+ *
+ * @param jobId Job Id
+ * @param logRetrievalScope Value for the retrieval type
+ * @param logRetrievalType Based on which filter criteria the log is retrieved
+ * @param writer writer to stream the log to
+ * @throws IOException
+ * @throws BaseEngineException
+ * @throws CommandException
+ */
+ public void streamLog(String jobId, String logRetrievalScope, String logRetrievalType, Writer writer)
+ throws IOException, BaseEngineException, CommandException {
+ XLogStreamer.Filter filter = new XLogStreamer.Filter();
+ filter.setParameter(DagXLogInfoService.JOB, jobId);
+ if (logRetrievalScope != null && logRetrievalType != null) {
+ if (logRetrievalType.equals(RestConstants.JOB_LOG_ACTION)) {
+ Set<String> actions = new HashSet<String>();
+ String[] list = logRetrievalScope.split(",");
+ for (String s : list) {
+ s = s.trim();
+ if (s.contains("-")) {
+ String[] range = s.split("-");
+ if (range.length != 2) {
+ throw new CommandException(ErrorCode.E0302, "format is wrong for action's range '" + s
+ + "'");
+ }
+ int start;
+ int end;
+ try {
+ start = Integer.parseInt(range[0].trim());
+ end = Integer.parseInt(range[1].trim());
+ if (start > end) {
+ throw new CommandException(ErrorCode.E0302, "format is wrong for action's range '" + s
+ + "'");
+ }
+ }
+ catch (NumberFormatException ne) {
+ throw new CommandException(ErrorCode.E0302, ne);
+ }
+ for (int i = start; i <= end; i++) {
+ actions.add(jobId + "@" + i);
+ }
+ }
+ else {
+ try {
+ Integer.parseInt(s);
+ }
+ catch (NumberFormatException ne) {
+ throw new CommandException(ErrorCode.E0302, "format is wrong for action id'" + s
+ + "'. Integer only.");
+ }
+ actions.add(jobId + "@" + s);
+ }
+ }
+
+ Iterator<String> actionsIterator = actions.iterator();
+ StringBuilder commaSeparatedActions = new StringBuilder("");
+ int commaRequired = 0;
+
+ while (actionsIterator.hasNext()) {
+ if (commaRequired == 1) {
+ commaSeparatedActions.append(",");
+ }
+ commaSeparatedActions.append(actionsIterator.next().toString());
+ commaRequired = 1;
+ }
+ filter.setParameter(DagXLogInfoService.ACTION, commaSeparatedActions.toString());
+ }
+ CoordinatorJobBean job = getCoordJobWithNoActionInfo(jobId);
+ Services.get().get(XLogService.class).streamLog(filter, job.getCreatedTime(), new Date(), writer);
+ }
+ else {
+ CoordinatorJobBean job = getCoordJobWithNoActionInfo(jobId);
+ Services.get().get(XLogService.class).streamLog(filter, job.getCreatedTime(), new Date(), writer);
+ }
+ }
+
/*
* (non-Javadoc)
*
@@ -338,7 +417,7 @@ public String dryrunSubmit(Configuration conf, boolean startJob) throws Coordina
}
else {
CoordSubmitCommand submit = new CoordSubmitCommand(true, conf, getAuthToken());
- jobId = submit.call();
+ jobId = submit.call();
}
return jobId;
}
@@ -435,11 +514,11 @@ public CoordinatorJobInfo getCoordJobs(String filterStr, int start, int len) thr
String[] pair = token.split("=");
if (pair.length != 2) {
throw new CoordinatorEngineException(ErrorCode.E0420, filter,
- "elements must be name=value pairs");
+ "elements must be name=value pairs");
}
if (!FILTER_NAMES.contains(pair[0])) {
throw new CoordinatorEngineException(ErrorCode.E0420, filter, XLog.format("invalid name [{0}]",
- pair[0]));
+ pair[0]));
}
if (pair[0].equals("status")) {
try {
View
25 core/src/main/java/org/apache/oozie/service/XLogService.java
@@ -37,7 +37,7 @@
import java.util.Date;
/**
- * Built-in service that initializes and manages Logging via Log4j.
+ * Built-in service that initializes and manages Logging via Log4j.
* <p/>
* Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>.
* <p/>
@@ -52,8 +52,8 @@
* <p/>
* If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled.
* <p/>
- * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>.
- * The default value is 10 seconds.
+ * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>. The default
+ * value is 10 seconds.
*/
public class XLogService implements Service, Instrumentable {
private static final String INSTRUMENTATION_GROUP = "logging";
@@ -148,10 +148,9 @@ public void init(Services services) throws ServiceException {
log = new XLog(LogFactory.getLog(getClass()));
log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION),
- BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo()
- .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty(
- BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo()
- .getProperty(BuildInfo.BUILD_VC_URL));
+ BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo()
+ .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty(
+ BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VC_URL));
String from = (fromClasspath) ? "CLASSPATH" : configPath;
String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec";
@@ -186,14 +185,15 @@ private void extractInfoForLogWebService(InputStream is) throws IOException {
}
String logFile = conf.get("log4j.appender.oozie.File");
if (logFile == null) {
- log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' appender");
+ log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' "
+ + "appender");
}
else {
logFile = logFile.trim();
int i = logFile.lastIndexOf("/");
if (i == -1) {
log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender",
- logFile);
+ logFile);
logOverWS = false;
}
else {
@@ -213,7 +213,7 @@ private void extractInfoForLogWebService(InputStream is) throws IOException {
}
else {
log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'",
- pattern);
+ pattern);
logOverWS = false;
}
}
@@ -258,8 +258,9 @@ public void destroy() {
}
/**
- * Instruments the log service. <p/> It sets instrumentation variables indicating the config file, reload interval
- * and if loaded from the classpath.
+ * Instruments the log service.
+ * <p/>
+ * It sets instrumentation variables indicating the config file, reload interval and if loaded from the classpath.
*
* @param instr instrumentation to use.
*/
View
35 core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java
@@ -29,6 +29,7 @@
import org.apache.oozie.CoordinatorActionInfo;
import org.apache.oozie.CoordinatorEngine;
import org.apache.oozie.CoordinatorEngineException;
+import org.apache.oozie.command.CommandException;
import org.apache.oozie.DagEngine;
import org.apache.oozie.DagEngineException;
import org.apache.oozie.ErrorCode;
@@ -160,6 +161,7 @@ else if (jobId.endsWith("-B")) {
* @throws XServletException
* @throws IOException
*/
+ @Override
protected void changeJob(HttpServletRequest request, HttpServletResponse response) throws XServletException,
IOException {
String jobId = getResourceName(request);
@@ -274,9 +276,9 @@ else if (jobId.endsWith("-B")) {
/**
* Start wf job
- *
+ *
* @param request servlet request
- * @param response servlet response
+ * @param response servlet response
* @throws XServletException
*/
private void startWorkflowJob(HttpServletRequest request, HttpServletResponse response) throws XServletException {
@@ -373,7 +375,7 @@ private void resumeCoordinatorJob(HttpServletRequest request, HttpServletRespons
/**
* Suspend a wf job
- *
+ *
* @param request servlet request
* @param response servlet response
* @throws XServletException
@@ -506,7 +508,7 @@ private void changeCoordinatorJob(HttpServletRequest request, HttpServletRespons
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
}
-
+
/**
* Change a bundle job
*
@@ -530,7 +532,7 @@ private void changeBundleJob(HttpServletRequest request, HttpServletResponse res
/**
* Rerun a wf job
- *
+ *
* @param request servlet request
* @param response servlet response
* @param conf configuration object
@@ -652,8 +654,8 @@ private JsonBean getWorkflowJob(HttpServletRequest request, HttpServletResponse
/**
* Get wf action info
- *
- * @param request servlet request
+ *
+ * @param request servlet request
* @param response servlet response
* @return JsonBean WorkflowActionBean
* @throws XServletException
@@ -677,7 +679,7 @@ private JsonBean getWorkflowAction(HttpServletRequest request, HttpServletRespon
/**
* Get coord job info
- *
+ *
* @param request servlet request
* @param response servlet response
* @return JsonBean CoordinatorJobBean
@@ -760,7 +762,7 @@ private JsonBean getCoordinatorAction(HttpServletRequest request, HttpServletRes
/**
* Get wf job definition
- *
+ *
* @param request servlet request
* @param response servlet response
* @return String wf definition
@@ -832,7 +834,7 @@ private String getCoordinatorJobDefinition(HttpServletRequest request, HttpServl
/**
* Stream wf job log
- *
+ *
* @param request servlet request
* @param response servlet response
* @throws XServletException
@@ -874,7 +876,7 @@ private void streamBundleJob(HttpServletRequest request, HttpServletResponse res
/**
* Stream coordinator job log
*
- * @param request servlet request
+ * @param request servlet request
* @param response servlet response
* @throws XServletException
* @throws IOException
@@ -884,14 +886,17 @@ private void streamCoordinatorJobLog(HttpServletRequest request, HttpServletResp
CoordinatorEngine coordEngine = Services.get().get(CoordinatorEngineService.class).getCoordinatorEngine(
getUser(request), getAuthToken(request));
-
String jobId = getResourceName(request);
-
+ String logRetrievalScope = request.getParameter(RestConstants.JOB_LOG_SCOPE_PARAM);
+ String logRetrievalType = request.getParameter(RestConstants.JOB_LOG_TYPE_PARAM);
try {
- coordEngine.streamLog(jobId, response.getWriter());
+ coordEngine.streamLog(jobId, logRetrievalScope, logRetrievalType, response.getWriter());
}
catch (BaseEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
+ catch (CommandException ex) {
+ throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
+ }
}
-}
+}
View
46 core/src/main/java/org/apache/oozie/util/XLogStreamer.java
@@ -45,7 +45,7 @@
private boolean noFilter;
private Pattern filterPattern;
- //TODO Patterns to be read from config file
+ // TODO Patterns to be read from config file
private static final String DEFAULT_REGEX = "[^\\]]*";
public static final String ALLOW_ALL_REGEX = "(.*)";
@@ -155,6 +155,21 @@ public void constructPattern() {
else {
sb.append("(.* - ");
for (int i = 0; i < parameters.size(); i++) {
+ if (parameters.get(i).equals("ACTION")) {
+ String[] actionsList = filterParams.get(parameters.get(i)).split(",");
+ sb.append("(");
+
+ for (int counter = 0; counter < actionsList.length; counter++) {
+ if (counter != 0) {
+ sb.append("|");
+ }
+ sb.append("ACTION" + "\\[");
+ sb.append(actionsList[counter] + "\\]");
+ }
+
+ sb.append(") ");
+ continue;
+ }
sb.append(parameters.get(i) + "\\[");
sb.append(filterParams.get(parameters.get(i)) + "\\] ");
}
@@ -196,6 +211,10 @@ public XLogStreamer(Filter logFilter, Writer logWriter, String logPath, String l
public void streamLog(Date startTime, Date endTime) throws IOException {
long startTimeMillis = 0;
long endTimeMillis;
+ String fileName;
+ InputStream ifs;
+ XLogReader logReader;
+
if (startTime != null) {
startTimeMillis = startTime.getTime();
}
@@ -208,9 +227,9 @@ public void streamLog(Date startTime, Date endTime) throws IOException {
File dir = new File(logPath);
ArrayList<FileInfo> fileList = getFileList(dir, startTimeMillis, endTimeMillis, logRotation, logFile);
for (int i = 0; i < fileList.size(); i++) {
- InputStream ifs;
+ fileName = fileList.get(i).getFileName();
ifs = new FileInputStream(fileList.get(i).getFileName());
- XLogReader logReader = new XLogReader(ifs, logFilter, logWriter);
+ logReader = new XLogReader(ifs, logFilter, logWriter);
logReader.processLog();
}
}
@@ -237,11 +256,13 @@ public long getModTime() {
public int compareTo(FileInfo fileInfo) {
long diff = this.modTime - fileInfo.modTime;
- if(diff > 0) {
+ if (diff > 0) {
return 1;
- } else if(diff < 0) {
+ }
+ else if (diff < 0) {
return -1;
- } else {
+ }
+ else {
return 0;
}
}
@@ -257,20 +278,23 @@ public int compareTo(FileInfo fileInfo) {
* @param logFile
* @return List of files to be streamed
*/
- private ArrayList<FileInfo> getFileList(File dir, long startTime, long endTime, long logRotationTime,
- String logFile) {
+ private ArrayList<FileInfo> getFileList(File dir, long startTime, long endTime, long logRotationTime, String logFile) {
String[] children = dir.list();
+ String fileName;
+ File file;
ArrayList<FileInfo> fileList = new ArrayList<FileInfo>();
+
if (children == null) {
return fileList;
}
else {
for (int i = 0; i < children.length; i++) {
- String filename = children[i];
- if (!filename.startsWith(logFile) && !filename.equals(logFile)) {
+ fileName = children[i];
+ file = new File(dir.getAbsolutePath(), fileName);
+ if (!fileName.startsWith(logFile) && !fileName.equals(logFile)) {
continue;
}
- File file = new File(dir.getAbsolutePath(), filename);
+ file = new File(dir.getAbsolutePath(), fileName);
long modTime = file.lastModified();
if (modTime < startTime) {
continue;
View
9 core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java
@@ -178,6 +178,14 @@ public void streamLog(String jobId, Writer writer) throws IOException, BaseEngin
writer.write(LOG);
}
+ @Override
+ public void streamLog(String jobId, String logRetrievalScope, String logRetrievalType, Writer writer)
+ throws IOException, BaseEngineException {
+ did = RestConstants.JOB_SHOW_LOG;
+ validateCoordinatorIdx(jobId);
+ writer.write(LOG);
+ }
+
private int validateCoordinatorIdx(String jobId) throws CoordinatorEngineException {
int idx = -1;
try {
@@ -274,5 +282,4 @@ private static JsonCoordinatorAction createDummyAction(int idx, String jobId) {
return action;
}
-
}
View
64 core/src/test/java/org/apache/oozie/util/TestXLogReader.java
@@ -38,13 +38,22 @@ public void testProcessLog() throws IOException {
FileWriter fw = new FileWriter(getTestCaseDir() + "/test.log");
StringBuilder sb = new StringBuilder();
- sb.append("2009-06-24 02:43:13,958 DEBUG _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
- sb.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] [org.apache.oozie.core.command.WorkflowRunnerCallable] released lock");
- sb.append("\n2009-06-24 02:43:13,986 WARN _L3_:539 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] JOB[14-200904160239--example-forkjoinwf] ACTION[-] Use GenericOptionsParser for parsing the arguments. \n_L3A_Applications should implement Tool for the same. \n_L3B_Multi line test");
- sb.append("\n2009-06-24 02:43:14,431 WARN _L4_:661 - No job jar file set. User classes may not be found. See JobConf(Class) or JobConf#setJar(String).");
- sb.append("\n2009-06-24 02:43:14,505 INFO _L5_:317 - USER[oozie] GROUP[oozie] TOKEN[-] APP[-] JOB[-] ACTION[-] Released Lock");
- sb.append("\n2009-06-24 02:43:19,344 DEBUG _L6_:323 - USER[oozie] GROUP[oozie] TOKEN[MYtoken] APP[-] JOB[-] ACTION[-] Number of pending signals to check [0]");
- sb.append("\n2009-06-24 02:43:29,151 DEBUG _L7_:323 - USER[-] GROUP[-] TOKEN[-] APP[-] JOB[-] ACTION[-] Number of pending actions [0] ");
+ sb.append("2009-06-24 02:43:13,958 DEBUG _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] End workflow state change");
+ sb.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] "
+ + "[org.apache.oozie.core.command.WorkflowRunnerCallable] " + "released lock");
+ sb.append("\n2009-06-24 02:43:13,986 WARN _L3_:539 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-forkjoinwf] ACTION[-] Use GenericOptionsParser for parsing "
+ + "the arguments. " + "\n_L3A_Applications should implement Tool for the same. \n_L3B_Multi line test");
+ sb.append("\n2009-06-24 02:43:14,431 WARN _L4_:661 - No job jar file set. User classes may not be found. "
+ + "See JobConf(Class) or JobConf#setJar(String).");
+ sb.append("\n2009-06-24 02:43:14,505 INFO _L5_:317 - USER[oozie] GROUP[oozie] TOKEN[-] APP[-] JOB[-] "
+ + "ACTION[-] " + "Released Lock");
+ sb.append("\n2009-06-24 02:43:19,344 DEBUG _L6_:323 - USER[oozie] GROUP[oozie] TOKEN[MYtoken] APP[-] "
+ + "JOB[-] ACTION[-] Number of pending signals to check [0]");
+ sb.append("\n2009-06-24 02:43:29,151 DEBUG _L7_:323 - USER[-] GROUP[-] TOKEN[-] APP[-] JOB[-] ACTION[-] "
+ + "Number of pending actions [0] ");
fw.write(sb.toString());
fw.close();
@@ -58,4 +67,45 @@ public void testProcessLog() throws IOException {
assertEquals(true, out[2].contains("_L3A_"));
assertEquals(true, out[3].contains("_L3B_"));
}
+
+ public void testProcessCoordinatorLogForActions() throws IOException {
+ XLogStreamer.Filter.reset();
+ XLogStreamer.Filter.defineParameter("USER");
+ XLogStreamer.Filter.defineParameter("GROUP");
+ XLogStreamer.Filter.defineParameter("TOKEN");
+ XLogStreamer.Filter.defineParameter("APP");
+ XLogStreamer.Filter.defineParameter("JOB");
+ XLogStreamer.Filter.defineParameter("ACTION");
+ XLogStreamer.Filter xf = new XLogStreamer.Filter();
+ xf.setParameter("JOB", "14-200904160239--example-C");
+ xf.setParameter("ACTION", "14-200904160239--example-C@1");
+ FileWriter fw = new FileWriter(getTestCaseDir() + "/test.log");
+ StringBuilder sb = new StringBuilder();
+ sb.append("2009-06-24 02:43:13,958 DEBUG _L1_:323 - USER[oozie] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-C] ACTION[14-200904160239--example-C@1] End workflow state change");
+ sb.append("\n2009-06-24 02:43:13,961 INFO _L2_:317 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-C] ACTION[14-200904160239--example-C@2] "
+ + "[org.apache.oozie.core.command.WorkflowRunnerCallable] released lock");
+ sb.append("\n2009-06-24 02:43:13,986 WARN _L3_:539 - USER[-] GROUP[-] TOKEN[-] APP[example-forkjoinwf] "
+ + "JOB[14-200904160239--example-C] ACTION[14-200904160239--example-C@2] Use GenericOptionsParser for "
+ + "parsing the arguments. \n_L3A_Applications should implement Tool for the same. \n_L3B_Multi line "
+ + "test");
+ sb.append("\n2009-06-24 02:43:14,431 WARN _L4_:661 - No job jar file set. User classes may not be found. "
+ + "See JobConf(Class) or JobConf#setJar(String).");
+ sb.append("\n2009-06-24 02:43:14,505 INFO _L5_:317 - USER[oozie] GROUP[oozie] TOKEN[-] APP[-] "
+ + "JOB[14-200904160239--example-C] ACTION[14-200904160239--example-C@1] Released Lock");
+ sb.append("\n2009-06-24 02:43:19,344 DEBUG _L6_:323 - USER[oozie] GROUP[oozie] TOKEN[MYtoken] APP[-] "
+ + "JOB[-] ACTION[-] Number of pending signals to check [0]");
+ sb.append("\n2009-06-24 02:43:29,151 DEBUG _L7_:323 - USER[-] GROUP[-] TOKEN[-] APP[-] JOB[-] "
+ + "ACTION[-] Number of pending actions [0] ");
+ fw.write(sb.toString());
+ fw.close();
+ StringWriter sw = new StringWriter();
+ XLogReader lr = new XLogReader(new FileInputStream(getTestCaseDir() + "/test.log"), xf, sw);
+ lr.processLog();
+ String[] matches = sw.toString().split("\n");
+ assertEquals(2, matches.length);
+ assertEquals(true, matches[0].contains("_L1_"));
+ assertEquals(true, matches[1].contains("_L5_"));
+ }
}
View
2  release-log.txt
@@ -5,6 +5,8 @@ OOZIE-18 Option to view Workflow job details from Coordinator job detail popup
OOZIE-17 Group column for coordinator jobs in Oozie Web Console
OOZIE-11 Adding Distcp Action.
OOZIE-6 Custom filters option and User information column added to Coordinator jobs section of Oozie Web Console
+OOZIE-5 Log retrieval for a Coordinator job with large number or actions
+OOZIE-7 Ability to view the log information corresponding to particular coordinator actions
OOZIE-124 Update documentation for job-type in WS API
OOZIE-81 Add an email action to Oozie
OOZIE-113 merge changes for OOZIE-101 to ActionEndXCommand
Please sign in to comment.
Something went wrong with that request. Please try again.