Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adds a setting to disable source output in slowlog #12655

Closed
wants to merge 3 commits into from
Closed

Adds a setting to disable source output in slowlog #12655

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c5eb3b1
Adds a setting to disable source output in slowlog
Aug 4, 2015
2965c4f
whitepsace, please squash
Aug 4, 2015
0bc03af
renamed variables, please squash
Aug 4, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,10 @@ public static State fromString(String state) {
public static final String SETTING_LEGACY_ROUTING_USE_TYPE = "index.legacy.routing.use_type";
public static final String SETTING_DATA_PATH = "index.data_path";
public static final String SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE = "index.shared_filesystem.recover_on_any_node";
public static final String SETTING_SLOW_LOGGING_SOURCE = "index.slow_logging.source";
public static final String INDEX_UUID_NA_VALUE = "_na_";


// hard-coded hash function as of 2.0
// older indices will read which hash function to use in their index settings
private static final HashFunction MURMUR3_HASH_FUNCTION = new Murmur3HashFunction();
Expand Down Expand Up @@ -202,6 +204,7 @@ public static State fromString(String state) {
private final org.apache.lucene.util.Version minimumCompatibleLuceneVersion;
private final HashFunction routingHashFunction;
private final boolean useTypeForRouting;
private final boolean logSourceInIndexSlowLog;

private IndexMetaData(String index, long version, State state, Settings settings, ImmutableOpenMap<String, MappingMetaData> mappings, ImmutableOpenMap<String, AliasMetaData> aliases, ImmutableOpenMap<String, Custom> customs) {
Preconditions.checkArgument(settings.getAsInt(SETTING_NUMBER_OF_SHARDS, null) != null, "must specify numberOfShards for index [" + index + "]");
Expand Down Expand Up @@ -256,6 +259,7 @@ private IndexMetaData(String index, long version, State state, Settings settings
}
}
useTypeForRouting = settings.getAsBoolean(SETTING_LEGACY_ROUTING_USE_TYPE, false);
logSourceInIndexSlowLog = settings.getAsBoolean(SETTING_SLOW_LOGGING_SOURCE, true);
}

public String index() {
Expand Down Expand Up @@ -388,6 +392,14 @@ public int getTotalNumberOfShards() {
return totalNumberOfShards();
}

public boolean logSourceInIndexSlowLog() {
return logSourceInIndexSlowLog;
}

public boolean getLogSourceInIndexSlowLog() {
return logSourceInIndexSlowLog;
}

public Settings settings() {
return settings;
}
Expand Down
42 changes: 27 additions & 15 deletions core/src/main/java/org/elasticsearch/index/indexing/IndexingSlowLog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.index.indexing;

import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
Expand All @@ -42,6 +43,7 @@ public final class IndexingSlowLog {
private long indexInfoThreshold;
private long indexDebugThreshold;
private long indexTraceThreshold;
private boolean indexSlowLoggingSource;

private String level;

Expand All @@ -62,7 +64,7 @@ public final class IndexingSlowLog {
this.indexInfoThreshold = indexSettings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO, TimeValue.timeValueNanos(-1)).nanos();
this.indexDebugThreshold = indexSettings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG, TimeValue.timeValueNanos(-1)).nanos();
this.indexTraceThreshold = indexSettings.getAsTime(INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE, TimeValue.timeValueNanos(-1)).nanos();

this.indexSlowLoggingSource = indexSettings.getAsBoolean(IndexMetaData.SETTING_SLOW_LOGGING_SOURCE, true);
this.level = indexSettings.get(INDEX_INDEXING_SLOWLOG_LEVEL, "TRACE").toUpperCase(Locale.ROOT);

this.indexLogger = Loggers.getLogger(INDEX_INDEXING_SLOWLOG_PREFIX +".index");
Expand Down Expand Up @@ -101,6 +103,11 @@ synchronized void onRefreshSettings(Settings settings) {
if (reformat != this.reformat) {
this.reformat = reformat;
}

boolean indexSlowLoggingSource = settings.getAsBoolean(IndexMetaData.SETTING_SLOW_LOGGING_SOURCE, true);
if(indexSlowLoggingSource != this.indexSlowLoggingSource) {
this.indexSlowLoggingSource = indexSlowLoggingSource;
}
}

void postIndex(Engine.Index index, long tookInNanos) {
Expand All @@ -113,25 +120,27 @@ void postCreate(Engine.Create create, long tookInNanos) {

private void postIndexing(ParsedDocument doc, long tookInNanos) {
if (indexWarnThreshold >= 0 && tookInNanos > indexWarnThreshold) {
indexLogger.warn("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat));
indexLogger.warn("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat, indexSlowLoggingSource));
} else if (indexInfoThreshold >= 0 && tookInNanos > indexInfoThreshold) {
indexLogger.info("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat));
indexLogger.info("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat, indexSlowLoggingSource));
} else if (indexDebugThreshold >= 0 && tookInNanos > indexDebugThreshold) {
indexLogger.debug("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat));
indexLogger.debug("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat, indexSlowLoggingSource));
} else if (indexTraceThreshold >= 0 && tookInNanos > indexTraceThreshold) {
indexLogger.trace("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat));
indexLogger.trace("{}", new SlowLogParsedDocumentPrinter(doc, tookInNanos, reformat, indexSlowLoggingSource));
}
}

final static class SlowLogParsedDocumentPrinter {
private final ParsedDocument doc;
private final long tookInNanos;
private final boolean reformat;
private final boolean indexSlowLoggingSource;

SlowLogParsedDocumentPrinter(ParsedDocument doc, long tookInNanos, boolean reformat) {
SlowLogParsedDocumentPrinter(ParsedDocument doc, long tookInNanos, boolean reformat, boolean indexSlowLoggingSource) {
this.doc = doc;
this.tookInNanos = tookInNanos;
this.reformat = reformat;
this.indexSlowLoggingSource = indexSlowLoggingSource;
}

@Override
Expand All @@ -141,18 +150,21 @@ public String toString() {
sb.append("type[").append(doc.type()).append("], ");
sb.append("id[").append(doc.id()).append("], ");
if (doc.routing() == null) {
sb.append("routing[], ");
sb.append("routing[] ");
} else {
sb.append("routing[").append(doc.routing()).append("], ");
sb.append("routing[").append(doc.routing()).append("] ");
}
if (doc.source() != null && doc.source().length() > 0) {
try {
sb.append("source[").append(XContentHelper.convertToJson(doc.source(), reformat)).append("]");
} catch (IOException e) {
sb.append("source[_failed_to_convert_]");

if(indexSlowLoggingSource) {
if (doc.source() != null && doc.source().length() > 0) {
try {
sb.append(", source[").append(XContentHelper.convertToJson(doc.source(), reformat)).append("]");
} catch (IOException e) {
sb.append(", source[_failed_to_convert_]");
}
} else {
sb.append(", source[]");
}
} else {
sb.append("source[]");
}
return sb.toString();
}
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import org.elasticsearch.cluster.settings.DynamicSettings;
import org.elasticsearch.cluster.settings.Validator;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.gateway.GatewayAllocator;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.indexing.IndexingSlowLog;
import org.elasticsearch.index.search.stats.SearchSlowLog;
Expand Down Expand Up @@ -73,6 +72,7 @@ public IndexDynamicSettingsModule() {
indexDynamicSettings.addDynamicSetting(IndexMetaData.SETTING_BLOCKS_METADATA);
indexDynamicSettings.addDynamicSetting(IndexMetaData.SETTING_SHARED_FS_ALLOW_RECOVERY_ON_ANY_NODE);
indexDynamicSettings.addDynamicSetting(IndexMetaData.SETTING_PRIORITY, Validator.NON_NEGATIVE_INTEGER);
indexDynamicSettings.addDynamicSetting(IndexMetaData.SETTING_SLOW_LOGGING_SOURCE, Validator.BOOLEAN);
indexDynamicSettings.addDynamicSetting(IndicesTTLService.INDEX_TTL_DISABLE_PURGE);
indexDynamicSettings.addDynamicSetting(IndexShard.INDEX_REFRESH_INTERVAL, Validator.TIME);
indexDynamicSettings.addDynamicSetting(PrimaryShardAllocator.INDEX_RECOVERY_INITIAL_SHARDS);
Expand Down