Skip to content
Permalink
Browse files
Enforce console logging for peon process (#12067)
Currently all Druid processes share the same log4j2 configuration file located in _common directory. Since peon processes are spawned by middle manager process, they derivate the environment variables from the middle manager. These variables include those in the log4j2.xml controlling to which file the logger writes the log.

But current task logging mechanism requires the peon processes to output the log to console so that the middle manager can redirect the console output to a file and upload this file to task log storage.

So, this PR imposes this requirement to peon processes, whatever the configuration is in the shared log4j2.xml, peon processes always write the log to console.
  • Loading branch information
FrankChen021 committed May 16, 2022
1 parent ff253fd commit c33ff1c745def3844c5f78007999a2bbdf676ba1
Showing 5 changed files with 431 additions and 2 deletions.
@@ -23,9 +23,12 @@ title: "Logging"
-->


Apache Druid processes will emit logs that are useful for debugging to the console. Druid processes also emit periodic metrics about their state. For more about metrics, see [Configuration](../configuration/index.md#enabling-metrics). Metric logs are printed to the console by default, and can be disabled with `-Ddruid.emitter.logging.logLevel=debug`.
Apache Druid processes will emit logs that are useful for debugging to log files.
These processes also emit periodic [metrics](../configuration/index.md#enabling-metrics) about their state.
Metric info logs can be disabled with `-Ddruid.emitter.logging.logLevel=debug`.

Druid uses [log4j2](http://logging.apache.org/log4j/2.x/) for logging. The default configuration file log4j2.xml ships with Druid under conf/druid/{config}/_common/log4j2.xml .
Druid uses [log4j2](http://logging.apache.org/log4j/2.x/) for logging.
The default configuration file log4j2.xml ships with Druid under conf/druid/{config}/_common/log4j2.xml .

By default, Druid uses `RollingRandomAccessFile` for rollover daily, and keeps log files up to 7 days.
If that's not suitable in your case, you could modify the log4j2.xml to meet your need.
@@ -67,6 +70,15 @@ An example log4j2.xml file is shown below:
</Configuration>
```

> NOTE:
> Although the log4j configuration file is shared with Druid's peon processes,
> the appenders in this file DO NOT take effect for peon processes for they always output logs to console.
> And middle managers are responsible to redirect the console output to task log files.
>
> But the logging levels settings take effect for these peon processes
> which means you can still configure loggers at different logging level for peon processes in this file.
>
## How to change log directory
By default, Druid outputs the logs to a directory `log` under the directory where Druid is launched from.
For example, if Druid is started from its `bin` directory, there will be a subdirectory `log` generated under `bin` directory to hold the log files.
@@ -0,0 +1,152 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.druid.indexing.common.tasklogs;

import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.core.Appender;
import org.apache.logging.log4j.core.Filter;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.ConsoleAppender;
import org.apache.logging.log4j.core.config.AppenderRef;
import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.ConfigurationFactory;
import org.apache.logging.log4j.core.config.ConfigurationSource;
import org.apache.logging.log4j.core.config.LoggerConfig;
import org.apache.logging.log4j.core.config.xml.XmlConfiguration;
import org.apache.logging.log4j.core.layout.PatternLayout;

import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

/**
* This class enforces console logging based on a user defined configuration.
* <p>
* For all loggers, this class ensure that only {@link ConsoleAppender} is applied for each of them.
* <p>
* The reason why this configuration is still based on a user's configuration is that
* user can still configure different logging levels for different loggers in that file.
*/
public class ConsoleLoggingEnforcementConfigurationFactory extends ConfigurationFactory
{
/**
* Valid file extensions for XML files.
*/
public static final String[] SUFFIXES = new String[]{".xml", "*"};

@Override
public String[] getSupportedTypes()
{
return SUFFIXES;
}

@Override
public Configuration getConfiguration(LoggerContext loggerContext, ConfigurationSource source)
{
return new OverrideConfiguration(loggerContext, source);
}

/**
* override the original configuration source to ensure only console appender is applied
*/
static class OverrideConfiguration extends XmlConfiguration
{
public OverrideConfiguration(final LoggerContext loggerContext, final ConfigurationSource configSource)
{
super(loggerContext, configSource);
}

@Override
protected void doConfigure()
{
super.doConfigure();

Appender consoleAppender = findConsoleAppender();
if (consoleAppender == null) {
// create a ConsoleAppender with default pattern if no console appender is configured in the configuration file
consoleAppender = ConsoleAppender.newBuilder()
.setName("_Injected_Console_Appender_")
.setLayout(PatternLayout.newBuilder()
.withPattern("%d{ISO8601} %p [%t] %c - %m%n")
.build())
.build();
}

List<LoggerConfig> loggerConfigList = new ArrayList<>();
loggerConfigList.add(this.getRootLogger());
loggerConfigList.addAll(this.getLoggers().values());

//
// For all logger configuration, check if its appender is ConsoleAppender.
// If not, replace it's appender to ConsoleAppender.
//
for (LoggerConfig logger : loggerConfigList) {
applyConsoleAppender(logger, consoleAppender);
}
}

@Nullable
private Appender findConsoleAppender()
{
for (Map.Entry<String, Appender> entry : this.getAppenders().entrySet()) {
Appender appender = entry.getValue();
if (appender instanceof ConsoleAppender) {
return appender;
}
}
return null;
}

/**
* remove all appenders from a logger and append a console appender to it
*/
private void applyConsoleAppender(LoggerConfig logger, Appender consoleAppender)
{
if (logger.getAppenderRefs().size() == 1
&& logger.getAppenderRefs().get(0).getRef().equals(consoleAppender.getName())) {
// this logger has only one appender and its the console appender
return;
}

Level level = Level.INFO;
Filter filter = null;

if (!logger.getAppenderRefs().isEmpty()) {
AppenderRef appenderRef = logger.getAppenderRefs().get(0);

// clear all appenders first
List<String> appendRefs = logger.getAppenderRefs()
.stream()
.map(AppenderRef::getRef)
.collect(Collectors.toList());
appendRefs.forEach(logger::removeAppender);

// use the first appender's definition
level = appenderRef.getLevel();
filter = appenderRef.getFilter();
}

// add ConsoleAppender to this logger
logger.addAppender(consoleAppender, level, filter);
}
}
}
@@ -47,6 +47,7 @@
import org.apache.druid.indexer.TaskStatus;
import org.apache.druid.indexing.common.config.TaskConfig;
import org.apache.druid.indexing.common.task.Task;
import org.apache.druid.indexing.common.tasklogs.ConsoleLoggingEnforcementConfigurationFactory;
import org.apache.druid.indexing.common.tasklogs.LogUtils;
import org.apache.druid.indexing.overlord.autoscaling.ScalingStats;
import org.apache.druid.indexing.overlord.config.ForkingTaskRunnerConfig;
@@ -339,6 +340,7 @@ public TaskStatus call()
command.add(
StringUtils.format("-Ddruid.task.executor.enableTlsPort=%s", node.isEnableTlsPort())
);
command.add(StringUtils.format("-Dlog4j2.configurationFactory=%s", ConsoleLoggingEnforcementConfigurationFactory.class.getName()));

// These are not enabled per default to allow the user to either set or not set them
// Users are highly suggested to be set in druid.indexer.runner.javaOpts

0 comments on commit c33ff1c

Please sign in to comment.