[FLINK-17166][dist] Modify the log4j-console.properties to also output logs into the files for WebUI

flink-dist/src/main/flink-bin/bin/flink-console.sh

@@ -58,7 +58,35 @@ esac
 
 FLINK_TM_CLASSPATH=`constructFlinkClassPath`
 
-log_setting=("-Dlog4j.configuration=file:${FLINK_CONF_DIR}/log4j-console.properties" "-Dlog4j.configurationFile=file:${FLINK_CONF_DIR}/log4j-console.properties" "-Dlogback.configurationFile=file:${FLINK_CONF_DIR}/logback-console.xml")
+if [ "$FLINK_IDENT_STRING" = "" ]; then
+    FLINK_IDENT_STRING="$USER"
+fi
+
+pid=$FLINK_PID_DIR/flink-$FLINK_IDENT_STRING-$SERVICE.pid
+mkdir -p "$FLINK_PID_DIR"
+# The lock needs to be released after use because this script is started foreground
+command -v flock >/dev/null 2>&1
+flock_exist=$?
+if [[ ${flock_exist} -eq 0 ]]; then
+    exec 200<"$FLINK_PID_DIR"
+    flock 200
+fi
+# Remove the pid file when all the processes are dead
+if [ -f "$pid" ]; then
+    all_dead=0
+    while read each_pid; do
+        # Check whether the process is still running
+        kill -0 $each_pid > /dev/null 2>&1
+        [[ $? -eq 0 ]] && all_dead=1
+    done < "$pid"
+    [ ${all_dead} -eq 0 ] && rm $pid
+fi
+id=$([ -f "$pid" ] && echo $(wc -l < "$pid") || echo "0")
+
+FLINK_LOG_PREFIX="${FLINK_LOG_DIR}/flink-${FLINK_IDENT_STRING}-${SERVICE}-${id}-${HOSTNAME}"
+log="${FLINK_LOG_PREFIX}.log"
+
+log_setting=("-Dlog.file=${log}" "-Dlog4j.configuration=file:${FLINK_CONF_DIR}/log4j-console.properties" "-Dlog4j.configurationFile=file:${FLINK_CONF_DIR}/log4j-console.properties" "-Dlogback.configurationFile=file:${FLINK_CONF_DIR}/logback-console.xml")
 
 JAVA_VERSION=$(${JAVA_RUN} -version 2>&1 | sed 's/.*version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
 
@@ -70,4 +98,11 @@ if [[ ${JAVA_VERSION} =~ ${IS_NUMBER} ]]; then
 fi
 
 echo "Starting $SERVICE as a console application on host $HOSTNAME."
+
+# Add the current process id to pid file
+echo $$ >> "$pid" 2>/dev/null
+
+# Release the lock because the java process runs in the foreground and would block other processes from modifying the pid file
+[[ ${flock_exist} -eq 0 ]] &&  flock -u 200
+
 exec $JAVA_RUN $JVM_ARGS ${FLINK_ENV_JAVA_OPTS} "${log_setting[@]}" -classpath "`manglePathList "$FLINK_TM_CLASSPATH:$INTERNAL_HADOOP_CLASSPATHS"`" ${CLASS_TO_RUN} "${ARGS[@]}"

flink-dist/src/main/flink-bin/conf/log4j-console.properties

@@ -19,6 +19,7 @@
 # This affects logging for both user code and Flink
 rootLogger.level = INFO
 rootLogger.appenderRef.console.ref = ConsoleAppender
+rootLogger.appenderRef.rolling.ref = RollingFileAppender
 
 # Uncomment this if you want to _only_ change Flink's logging
 #log4j.logger.org.apache.flink=INFO
@@ -41,6 +42,20 @@ appender.console.type = CONSOLE
 appender.console.layout.type = PatternLayout
 appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
 
+# Log all infos in the given rolling file
+appender.rolling.name = RollingFileAppender
+appender.rolling.type = RollingFile
+appender.rolling.append = false
+appender.rolling.fileName = ${sys:log.file}
+appender.rolling.filePattern = ${sys:log.file}.%i
+appender.rolling.layout.type = PatternLayout
+appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+appender.rolling.policies.type = Policies
+appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.rolling.policies.size.size=100MB
+appender.rolling.strategy.type = DefaultRolloverStrategy
+appender.rolling.strategy.max = 10
+
 # Suppress the irrelevant (wrong) warnings from the Netty channel handler
 logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
 logger.netty.level = OFF

flink-dist/src/main/flink-bin/conf/logback-console.xml

@@ -23,34 +23,42 @@
         </encoder>
     </appender>
 
+    <appender name="rolling" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${log.file}</file>
+        <append>false</append>
+
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${log.file}.%i</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+
+        <encoder>
+            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{60} %X{sourceThread} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
     <!-- This affects logging for both user code and Flink -->
     <root level="INFO">
         <appender-ref ref="console"/>
+        <appender-ref ref="rolling"/>
     </root>
 
     <!-- Uncomment this if you want to only change Flink's logging -->
-    <!--<logger name="org.apache.flink" level="INFO">-->
-        <!--<appender-ref ref="console"/>-->
-    <!--</logger>-->
+    <!--<logger name="org.apache.flink" level="INFO"/>-->
 
     <!-- The following lines keep the log level of common libraries/connectors on
          log level INFO. The root logger does not override this. You have to manually
          change the log levels here. -->
-    <logger name="akka" level="INFO">
-        <appender-ref ref="console"/>
-    </logger>
-    <logger name="org.apache.kafka" level="INFO">
-        <appender-ref ref="console"/>
-    </logger>
-    <logger name="org.apache.hadoop" level="INFO">
-        <appender-ref ref="console"/>
-    </logger>
-    <logger name="org.apache.zookeeper" level="INFO">
-        <appender-ref ref="console"/>
-    </logger>
+    <logger name="akka" level="INFO"/>
+    <logger name="org.apache.kafka" level="INFO"/>
+    <logger name="org.apache.hadoop" level="INFO"/>
+    <logger name="org.apache.zookeeper" level="INFO"/>
 
     <!-- Suppress the irrelevant (wrong) warnings from the Netty channel handler -->
-    <logger name="org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline" level="ERROR">
-        <appender-ref ref="console"/>
-    </logger>
+    <logger name="org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline" level="ERROR"/>
 </configuration>