Fix Json out put of json console (remove duplicate basic messages and abide by consoleloglevel)

dev/com.ibm.ws.logging/src/com/ibm/ws/logging/data/GenericData.java

@@ -11,6 +11,7 @@
 package com.ibm.ws.logging.data;
 
 import java.util.ArrayList;
+import java.util.logging.Level;
 
 public class GenericData {
 
@@ -20,6 +21,10 @@ public class GenericData {
 
     private String sourceType;
 
+    private Level logRecordLevel = null;
+
+    private String loggerName = null;
+
     public GenericData() {
         pairs = new ArrayList<Pair>();
     }
@@ -50,6 +55,22 @@ public void setSourceType(String sourceType) {
         this.sourceType = sourceType;
     }
 
+    public void setLogRecordLevel(Level logRecordLevel) {
+        this.logRecordLevel = logRecordLevel;
+    }
+
+    public Level getLogRecordLevel() {
+        return logRecordLevel;
+    }
+
+    public void setLoggerName(String loggerName) {
+        this.loggerName = loggerName;
+    }
+
+    public String getLoggerName() {
+        return loggerName;
+    }
+
     //Method created to accomodate some tests, must remove down the line
     public String getMessageID() {
         for (Pair p : pairs) {

dev/com.ibm.ws.logging/src/com/ibm/ws/logging/internal/impl/ConsoleLogHandler.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2017 IBM Corporation and others.
+ * Copyright (c) 2017, 2018 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -11,9 +11,12 @@
 package com.ibm.ws.logging.internal.impl;
 
 import java.util.List;
+import java.util.logging.Level;
 
+import com.ibm.websphere.logging.WsLevel;
 import com.ibm.ws.logging.collector.CollectorConstants;
 import com.ibm.ws.logging.collector.Formatter;
+import com.ibm.ws.logging.data.GenericData;
 import com.ibm.ws.logging.internal.impl.BaseTraceService.SystemLogHolder;
 import com.ibm.wsspi.collector.manager.SynchronousHandler;
 
@@ -25,6 +28,9 @@ public class ConsoleLogHandler extends JsonLogHandler implements SynchronousHand
     public static final String COMPONENT_NAME = "com.ibm.ws.logging.internal.impl.ConsoleLogHandler";
     private SystemLogHolder sysLogHolder;
 
+    private Level consoleLogLevel;
+    private boolean copySystemStreams;
+
     public ConsoleLogHandler(String serverName, String wlpUserDir, List<String> sourcesList) {
         super(serverName, wlpUserDir, sourcesList);
     }
@@ -41,14 +47,74 @@ public void synchronousWrite(Object event) {
          * Knowing that it is a *Data object, we can figure what type of source it is.
          */
         String evensourcetType = getSourceTypeFromDataObject(event);
+        int logLevelValue = Integer.MIN_VALUE;
+        String loggerName = null;
+        String sourceType = null;
+        if (event instanceof GenericData) {
+            GenericData genData = (GenericData) event;
+            loggerName = genData.getLoggerName();
+            sourceType = genData.getSourceType();
+            Level logRecordLevel = genData.getLogRecordLevel();
+            if (logRecordLevel != null) {
+                logLevelValue = logRecordLevel.intValue();
+            }
+        }
+
         String messageOutput = (String) formatEvent(evensourcetType, CollectorConstants.MEMORY, event, null, MAXFIELDLENGTH);
         synchronized (this) {
-            sysLogHolder.getOriginalStream().println(messageOutput);
+
+            //Write out accessLog or ffdc or trace
+            if (sourceType.equals(CollectorConstants.ACCESS_LOG_SOURCE) ||
+                sourceType.equals(CollectorConstants.TRACE_SOURCE) ||
+                sourceType.equals(CollectorConstants.FFDC_SOURCE)) {
+                sysLogHolder.getOriginalStream().println(messageOutput);
+                return;
+            }
+
+            /*
+             * We only allow two types of console messages to go through:
+             *
+             * 1. CopySystemStreams is true AND this message came from BaseTraceService.TrOutputStream which exhibit the following characteristics:
+             * - LogLevel of WsLevel.CONFIG
+             * - LoggerName of LoggingConstants.SYSTEM_OUT (i.e SystemOut) OR loggerNameLoggingConstants.SYSTEM_ERR (i.e. SystemErr)
+             * OR
+             * 2. Either this message is greater than or equal to consoleLogLevel (i.e from publishLogRecord)
+             *
+             */
+
+            if (copySystemStreams &&
+                logLevelValue == WsLevel.CONFIG.intValue() &&
+                (loggerName.equalsIgnoreCase(LoggingConstants.SYSTEM_OUT) || loggerName.equalsIgnoreCase(LoggingConstants.SYSTEM_ERR))) {
+                sysLogHolder.getOriginalStream().println(messageOutput);
+                return;
+            }
+
+            if (logLevelValue >= consoleLogLevel.intValue()) {
+                sysLogHolder.getOriginalStream().println(messageOutput);
+                return;
+            }
+
         }
     }
 
     @Override
     public void setWriter(Object writer) {
         this.sysLogHolder = (SystemLogHolder) writer;
     }
+
+    public Level getConsoleLogLevel() {
+        return consoleLogLevel;
+    }
+
+    public void setConsoleLogLevel(Level consoleLogLevel) {
+        this.consoleLogLevel = consoleLogLevel;
+    }
+
+    public boolean getCopySystemStreams() {
+        return copySystemStreams;
+    }
+
+    public void setCopySystemStreams(boolean copySystemStreams) {
+        this.copySystemStreams = copySystemStreams;
+    }
 }

dev/com.ibm.ws.logging/src/com/ibm/ws/logging/internal/impl/JsonTraceService.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2017 IBM Corporation and others.
+ * Copyright (c) 2017, 2018 IBM Corporation and others.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
  * which accompanies this distribution, and is available at
@@ -111,6 +111,9 @@ public synchronized void update(LogProviderConfig config) {
             collectorMgrPipelineUtils.setConsoleHandler(consoleLogHandler);
             consoleLogHandler.setWriter(systemOut);
         }
+        //These two must be set here so that it can get the latest config for *every* update call
+        consoleLogHandler.setConsoleLogLevel(consoleLogLevel);
+        consoleLogHandler.setCopySystemStreams(copySystemStreams);
 
         /*
          * If messageFormat has been configured to 'basic' - ensure that we are not connecting conduits/bufferManagers to the handler
@@ -224,7 +227,7 @@ public void echo(SystemLogHolder holder, LogRecord logRecord) {
             // preserve System.out vs. System.err
             publishTraceLogRecord(holder, logRecord, NULL_ID, NULL_FORMATTED_MSG, NULL_FORMATTED_MSG);
         } else {
-            if (copySystemStreams) {
+            if (copySystemStreams && !isConsoleJsonConfigured) {
                 // Tee to console.log if we are copying System.out and System.err to system streams.
                 writeFilteredStreamOutput(holder, logRecord);
             }

dev/com.ibm.ws.logging/src/com/ibm/ws/logging/source/LogSource.java

@@ -181,6 +181,8 @@ public GenericData parse(RoutedMessage routedMessage) {
         }
         genData.addPair("message", msgBldr.toString());
         genData.setSourceType(sourceName);
+        genData.setLogRecordLevel(logRecord.getLevel());
+        genData.setLoggerName(logRecord.getLoggerName());
 
         return genData;