New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add settings to control size and count of warning headers in responses #28427
Changes from 1 commit
9a3f8ad
0fa44fb
48a5768
63dc6b8
e2310f0
9e6de75
4e2fc02
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -23,10 +23,16 @@ | |
import org.elasticsearch.common.io.stream.StreamInput; | ||
import org.elasticsearch.common.io.stream.StreamOutput; | ||
import org.elasticsearch.common.io.stream.Writeable; | ||
import org.elasticsearch.common.logging.DeprecationLogger; | ||
import org.elasticsearch.common.logging.ESLoggerFactory; | ||
import org.elasticsearch.common.settings.Setting; | ||
import org.elasticsearch.common.settings.Setting.Property; | ||
import org.elasticsearch.common.settings.Settings; | ||
import org.elasticsearch.common.unit.ByteSizeValue; | ||
import org.elasticsearch.http.HttpTransportSettings; | ||
|
||
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT; | ||
import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE; | ||
|
||
import java.io.Closeable; | ||
import java.io.IOException; | ||
|
@@ -39,13 +45,14 @@ | |
import java.util.Set; | ||
import java.util.concurrent.CancellationException; | ||
import java.util.concurrent.ExecutionException; | ||
import java.util.concurrent.FutureTask; | ||
import java.util.concurrent.RunnableFuture; | ||
import java.util.concurrent.atomic.AtomicBoolean; | ||
import java.util.function.Function; | ||
import java.util.function.Supplier; | ||
import java.util.stream.Collectors; | ||
import java.util.stream.Stream; | ||
import java.nio.charset.StandardCharsets; | ||
|
||
|
||
/** | ||
* A ThreadContext is a map of string headers and a transient map of keyed objects that are associated with | ||
|
@@ -81,6 +88,8 @@ public final class ThreadContext implements Closeable, Writeable { | |
private static final ThreadContextStruct DEFAULT_CONTEXT = new ThreadContextStruct(); | ||
private final Map<String, String> defaultHeader; | ||
private final ContextThreadLocal threadLocal; | ||
private static volatile int maxWrnHeaderCount; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's use traditional Java variable style names here (e.g., |
||
private static volatile long maxWrnHeaderSize; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
|
||
/** | ||
* Creates a new ThreadContext instance | ||
|
@@ -98,13 +107,23 @@ public ThreadContext(Settings settings) { | |
this.defaultHeader = Collections.unmodifiableMap(defaultHeader); | ||
} | ||
threadLocal = new ContextThreadLocal(); | ||
maxWrnHeaderCount = SETTING_HTTP_MAX_WARNING_HEADER_COUNT.get(settings); | ||
maxWrnHeaderSize = SETTING_HTTP_MAX_WARNING_HEADER_SIZE.get(settings).getBytes(); | ||
} | ||
|
||
@Override | ||
public void close() throws IOException { | ||
threadLocal.close(); | ||
} | ||
|
||
public static void setMaxWarningHeaderCount(int newMaxWrnHeaderCount){ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: spacing between the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Make the method parameter |
||
maxWrnHeaderCount = newMaxWrnHeaderCount; | ||
} | ||
|
||
public static void setMaxWarningHeaderSize(ByteSizeValue newMaxWarningHeaderSize){ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Make the method parameter There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
maxWrnHeaderSize = newMaxWarningHeaderSize.getBytes(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why do we accept a |
||
} | ||
|
||
/** | ||
* Removes the current context and resets a default context. The removed context can be | ||
* restored when closing the returned {@link StoredContext} | ||
|
@@ -359,7 +378,8 @@ private static final class ThreadContextStruct { | |
private final Map<String, Object> transientHeaders; | ||
private final Map<String, List<String>> responseHeaders; | ||
private final boolean isSystemContext; | ||
|
||
private long wrnHeadersSize; //saving current warning headers' size not to recalculate the size with every new warning header | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
private boolean isWrnLmtReached; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this field strictly needed? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think yes. this field once the limit is reached ensures that we don't check and log "warnings limit reached" again and again with a new warning header. |
||
private ThreadContextStruct(StreamInput in) throws IOException { | ||
final int numRequest = in.readVInt(); | ||
Map<String, String> requestHeaders = numRequest == 0 ? Collections.emptyMap() : new HashMap<>(numRequest); | ||
|
@@ -371,6 +391,8 @@ private ThreadContextStruct(StreamInput in) throws IOException { | |
this.responseHeaders = in.readMapOfLists(StreamInput::readString, StreamInput::readString); | ||
this.transientHeaders = Collections.emptyMap(); | ||
isSystemContext = false; // we never serialize this it's a transient flag | ||
wrnHeadersSize = 0L; | ||
isWrnLmtReached = false; | ||
} | ||
|
||
private ThreadContextStruct setSystemContext() { | ||
|
@@ -387,6 +409,20 @@ private ThreadContextStruct(Map<String, String> requestHeaders, | |
this.responseHeaders = responseHeaders; | ||
this.transientHeaders = transientHeaders; | ||
this.isSystemContext = isSystemContext; | ||
this.wrnHeadersSize = 0L; | ||
isWrnLmtReached = false; | ||
} | ||
|
||
private ThreadContextStruct(Map<String, String> requestHeaders, | ||
Map<String, List<String>> responseHeaders, | ||
Map<String, Object> transientHeaders, boolean isSystemContext, | ||
long wrnHeadersSize, boolean isWrnLmtReached) { | ||
this.requestHeaders = requestHeaders; | ||
this.responseHeaders = responseHeaders; | ||
this.transientHeaders = transientHeaders; | ||
this.isSystemContext = isSystemContext; | ||
this.wrnHeadersSize = wrnHeadersSize; | ||
this.isWrnLmtReached = isWrnLmtReached; | ||
} | ||
|
||
/** | ||
|
@@ -442,6 +478,19 @@ private ThreadContextStruct putResponseHeaders(Map<String, List<String>> headers | |
|
||
private ThreadContextStruct putResponse(final String key, final String value, final Function<String, String> uniqueValue) { | ||
assert value != null; | ||
long curWrnHeaderSize = 0; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
//check if we can add another warning header (max count or size within limits) | ||
if (key.equals("Warning")) { | ||
if (isWrnLmtReached) return this; //can't add warning headers - limit reached | ||
if (maxWrnHeaderCount != -1) { //if count is NOT unbounded, check its limits | ||
int wrnHeaderCount = this.responseHeaders.containsKey("Warning") ? this.responseHeaders.get("Warning").size() : 0; | ||
if (wrnHeaderCount >= maxWrnHeaderCount) return addWrnLmtReachedHeader(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Take care, the warning headers are de-duplicated below. The accounting should happen after de-duplication, otherwise we truncate when a warning header would not have been added because it is a duplicate of an existing warning header. |
||
} | ||
if (maxWrnHeaderSize != -1) { //if size is NOT unbounded, check its limits | ||
curWrnHeaderSize = "Warning".getBytes(StandardCharsets.UTF_8).length + value.getBytes(StandardCharsets.UTF_8).length; | ||
if ((wrnHeadersSize + curWrnHeaderSize) > maxWrnHeaderSize) return addWrnLmtReachedHeader(); | ||
} | ||
} | ||
|
||
final Map<String, List<String>> newResponseHeaders = new HashMap<>(this.responseHeaders); | ||
final List<String> existingValues = newResponseHeaders.get(key); | ||
|
@@ -460,8 +509,37 @@ private ThreadContextStruct putResponse(final String key, final String value, fi | |
} else { | ||
newResponseHeaders.put(key, Collections.singletonList(value)); | ||
} | ||
return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, | ||
isSystemContext, wrnHeadersSize + curWrnHeaderSize, isWrnLmtReached); | ||
} | ||
|
||
return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext); | ||
//replace last warning header(s) with "headers limit reached" warning | ||
//respecting limitations on headers size if it is set by user | ||
private ThreadContextStruct addWrnLmtReachedHeader(){ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The user already expects if they have set this setting that warning headers will be truncated. Therefore, I don't think we should do this, losing a warning header (or more) at the expense of a warning header saying that there are more warnings that can be found in the logs. Instead I think that we should log (in the main Elasticsearch log) when we truncate. |
||
if ((maxWrnHeaderSize == 0) || (maxWrnHeaderCount ==0)) //can't even add "headers limit reached" warning | ||
return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, | ||
isSystemContext, wrnHeadersSize, true); | ||
final Map<String, List<String>> newResponseHeaders = new HashMap<>(this.responseHeaders); | ||
final List<String> wrns = new ArrayList<>(newResponseHeaders.get("Warning")); | ||
final String lastWrnMessage = DeprecationLogger.formatWarning( | ||
"There were more warnings, but they were dropped as [" + | ||
HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey() + "] or [" + | ||
HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey() + "] were reached!"); | ||
|
||
if (maxWrnHeaderSize > 0) { | ||
final long wrnSize = "Warning".getBytes(StandardCharsets.UTF_8).length; | ||
wrnHeadersSize = wrnHeadersSize + wrnSize + lastWrnMessage.getBytes(StandardCharsets.UTF_8).length; | ||
do { | ||
String wrn = wrns.remove(wrns.size() - 1); | ||
wrnHeadersSize = wrnHeadersSize - wrnSize - wrn.getBytes(StandardCharsets.UTF_8).length; | ||
} while(wrnHeadersSize > maxWrnHeaderSize); | ||
} else { //we don't care about size as it is unbounded | ||
wrns.remove(wrns.size() - 1); | ||
} | ||
wrns.add(lastWrnMessage); | ||
newResponseHeaders.put("Warning", Collections.unmodifiableList(wrns)); | ||
return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, | ||
isSystemContext, wrnHeadersSize, true); | ||
} | ||
|
||
private ThreadContextStruct putTransient(String key, Object value) { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think that the documentation for these settings should be moved to the HTTP docs.