Skip to content

Commit

Permalink
Add whitelist to watcher HttpClient (#36817)
Browse files Browse the repository at this point in the history 
This adds a configurable whitelist to the HTTP client in watcher. By
default every URL is allowed to retain BWC. A dynamically configurable
setting named "xpack.http.whitelist" was added that allows to
configure an array of URLs, which can also contain simple regexes.

Closes #29937
  • Loading branch information
@spinscale
spinscale committed Jan 11, 2019
1 parent 37493c2 commit bbd0930
Show file tree
Hide file tree
Showing 10 changed files with 286 additions and 35 deletions.
8 changes: 8 additions & 0 deletions docs/reference/settings/notification-settings.asciidoc
View file
Expand Up @@ -64,6 +64,14 @@ request is aborted.
Specifies the maximum size an HTTP response is allowed to have, defaults to
`10mb`, the maximum configurable value is `50mb`.

`xpack.http.whitelist`::
A list of URLs, that the internal HTTP client is allowed to connect to. This
client is used in the HTTP input, the webhook, the slack, pagerduty, hipchat
and jira actions. This setting can be updated dynamically. It defaults to `*`
allowing everything. Note: If you configure this setting and you are using one
of the slack/pagerduty/hipchat actions, you have to ensure that the
corresponding endpoints are whitelisted as well.

[[ssl-notification-settings]]
:ssl-prefix: xpack.http
:component: {watcher}
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java
View file
Expand Up @@ -273,7 +273,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
new WatcherIndexTemplateRegistry(clusterService, threadPool, client);

// http client
httpClient = new HttpClient(settings, getSslService(), cryptoService);
httpClient = new HttpClient(settings, getSslService(), cryptoService, clusterService);

// notification
EmailService emailService = new EmailService(settings, cryptoService, clusterService.getClusterSettings());
Expand Down
84 changes: 82 additions & 2 deletions .../plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java
View file
Expand Up @@ -8,7 +8,9 @@
import org.apache.http.Header;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.NameValuePair;
import org.apache.http.ProtocolException;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.Credentials;
import org.apache.http.auth.UsernamePasswordCredentials;
Expand All @@ -19,6 +21,7 @@
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.methods.HttpRequestWrapper;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.utils.URIUtils;
import org.apache.http.client.utils.URLEncodedUtils;
Expand All @@ -31,11 +34,20 @@
import org.apache.http.impl.client.BasicAuthCache;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HttpContext;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.MinimizationOperations;
import org.apache.lucene.util.automaton.Operations;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
Expand All @@ -59,6 +71,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;

public class HttpClient implements Closeable {

Expand All @@ -69,20 +82,29 @@ public class HttpClient implements Closeable {
private static final int MAX_CONNECTIONS = 500;
private static final Logger logger = LogManager.getLogger(HttpClient.class);

private final AtomicReference<CharacterRunAutomaton> whitelistAutomaton = new AtomicReference<>();
private final CloseableHttpClient client;
private final HttpProxy settingsProxy;
private final TimeValue defaultConnectionTimeout;
private final TimeValue defaultReadTimeout;
private final ByteSizeValue maxResponseSize;
private final CryptoService cryptoService;
private final SSLService sslService;

public HttpClient(Settings settings, SSLService sslService, CryptoService cryptoService) {
public HttpClient(Settings settings, SSLService sslService, CryptoService cryptoService, ClusterService clusterService) {
this.defaultConnectionTimeout = HttpSettings.CONNECTION_TIMEOUT.get(settings);
this.defaultReadTimeout = HttpSettings.READ_TIMEOUT.get(settings);
this.maxResponseSize = HttpSettings.MAX_HTTP_RESPONSE_SIZE.get(settings);
this.settingsProxy = getProxyFromSettings(settings);
this.cryptoService = cryptoService;
this.sslService = sslService;

setWhitelistAutomaton(HttpSettings.HOSTS_WHITELIST.get(settings));
clusterService.getClusterSettings().addSettingsUpdateConsumer(HttpSettings.HOSTS_WHITELIST, this::setWhitelistAutomaton);
this.client = createHttpClient();
}

private CloseableHttpClient createHttpClient() {
HttpClientBuilder clientBuilder = HttpClientBuilder.create();

// ssl setup
Expand All @@ -95,8 +117,48 @@ public HttpClient(Settings settings, SSLService sslService, CryptoService crypto
clientBuilder.evictExpiredConnections();
clientBuilder.setMaxConnPerRoute(MAX_CONNECTIONS);
clientBuilder.setMaxConnTotal(MAX_CONNECTIONS);
clientBuilder.setRedirectStrategy(new DefaultRedirectStrategy() {
@Override
public boolean isRedirected(org.apache.http.HttpRequest request, org.apache.http.HttpResponse response,
HttpContext context) throws ProtocolException {
boolean isRedirected = super.isRedirected(request, response, context);
if (isRedirected) {
String host = response.getHeaders("Location")[0].getValue();
if (isWhitelisted(host) == false) {
throw new ElasticsearchException("host [" + host + "] is not whitelisted in setting [" +
HttpSettings.HOSTS_WHITELIST.getKey() + "], will not redirect");
}
}

return isRedirected;
}
});

clientBuilder.addInterceptorFirst((HttpRequestInterceptor) (request, context) -> {
if (request instanceof HttpRequestWrapper == false) {
throw new ElasticsearchException("unable to check request [{}/{}] for white listing", request,
request.getClass().getName());
}

HttpRequestWrapper wrapper = ((HttpRequestWrapper) request);
final String host;
if (wrapper.getTarget() != null) {
host = wrapper.getTarget().toURI();
} else {
host = wrapper.getOriginal().getRequestLine().getUri();
}

client = clientBuilder.build();
if (isWhitelisted(host) == false) {
throw new ElasticsearchException("host [" + host + "] is not whitelisted in setting [" +
HttpSettings.HOSTS_WHITELIST.getKey() + "], will not connect");
}
});

return clientBuilder.build();
}

private void setWhitelistAutomaton(List<String> whiteListedHosts) {
whitelistAutomaton.set(createAutomaton(whiteListedHosts));
}

public HttpResponse execute(HttpRequest request) throws IOException {
Expand Down Expand Up @@ -285,6 +347,24 @@ final class HttpMethodWithEntity extends HttpEntityEnclosingRequestBase {
public String getMethod() {
return methodName;
}

}

private boolean isWhitelisted(String host) {
return whitelistAutomaton.get().run(host);
}

private static final CharacterRunAutomaton MATCH_ALL_AUTOMATON = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton("*"));
// visible for testing
static CharacterRunAutomaton createAutomaton(List<String> whiteListedHosts) {
if (whiteListedHosts.isEmpty()) {
// the default is to accept everything, this should change in the next major version, being 8.0
// we could emit depreciation warning here, if the whitelist is empty
return MATCH_ALL_AUTOMATON;
}

Automaton whiteListAutomaton = Regex.simpleMatchToAutomaton(whiteListedHosts.toArray(Strings.EMPTY_ARRAY));
whiteListAutomaton = MinimizationOperations.minimize(whiteListAutomaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
return new CharacterRunAutomaton(whiteListAutomaton);
}
}
29 changes: 15 additions & 14 deletions ...plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java
View file
Expand Up @@ -35,6 +35,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;

import static java.util.Collections.emptyMap;
import static java.util.Collections.unmodifiableMap;
Expand Down Expand Up @@ -154,10 +155,8 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params toX
builder.field(Field.PARAMS.getPreferredName(), this.params);
}
if (headers.isEmpty() == false) {
if (WatcherParams.hideSecrets(toXContentParams) && headers.containsKey("Authorization")) {
Map<String, String> sanitizedHeaders = new HashMap<>(headers);
sanitizedHeaders.put("Authorization", WatcherXContentParser.REDACTED_PASSWORD);
builder.field(Field.HEADERS.getPreferredName(), sanitizedHeaders);
if (WatcherParams.hideSecrets(toXContentParams)) {
builder.field(Field.HEADERS.getPreferredName(), sanitizeHeaders(headers));
} else {
builder.field(Field.HEADERS.getPreferredName(), headers);
}
Expand All @@ -184,6 +183,15 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params toX
return builder.endObject();
}

private Map<String, String> sanitizeHeaders(Map<String, String> headers) {
if (headers.containsKey("Authorization") == false) {
return headers;
}
Map<String, String> sanitizedHeaders = new HashMap<>(headers);
sanitizedHeaders.put("Authorization", WatcherXContentParser.REDACTED_PASSWORD);
return sanitizedHeaders;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand Down Expand Up @@ -220,16 +228,9 @@ public String toString() {
sb.append("port=[").append(port).append("], ");
sb.append("path=[").append(path).append("], ");
if (!headers.isEmpty()) {
sb.append(", headers=[");
boolean first = true;
for (Map.Entry<String, String> header : headers.entrySet()) {
if (!first) {
sb.append(", ");
}
sb.append("[").append(header.getKey()).append(": ").append(header.getValue()).append("]");
first = false;
}
sb.append("], ");
sb.append(sanitizeHeaders(headers).entrySet().stream()
.map(header -> header.getKey() + ": " + header.getValue())
.collect(Collectors.joining(", ", "headers=[", "], ")));
}
if (auth != null) {
sb.append("auth=[").append(BasicAuth.TYPE).append("], ");
Expand Down
5 changes: 5 additions & 0 deletions ...lugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpSettings.java
View file
Expand Up @@ -13,7 +13,9 @@
import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.function.Function;

/**
* Handles the configuration and parsing of settings for the <code>xpack.http.</code> prefix
Expand All @@ -36,6 +38,8 @@ public class HttpSettings {
static final Setting<String> PROXY_HOST = Setting.simpleString(PROXY_HOST_KEY, Property.NodeScope);
static final Setting<String> PROXY_SCHEME = Setting.simpleString(PROXY_SCHEME_KEY, Scheme::parse, Property.NodeScope);
static final Setting<Integer> PROXY_PORT = Setting.intSetting(PROXY_PORT_KEY, 0, 0, 0xFFFF, Property.NodeScope);
static final Setting<List<String>> HOSTS_WHITELIST = Setting.listSetting("xpack.http.whitelist", Collections.singletonList("*"),
Function.identity(), Property.NodeScope, Property.Dynamic);

static final Setting<ByteSizeValue> MAX_HTTP_RESPONSE_SIZE = Setting.byteSizeSetting("xpack.http.max_response_size",
new ByteSizeValue(10, ByteSizeUnit.MB), // default
Expand All @@ -54,6 +58,7 @@ public static List<? extends Setting<?>> getSettings() {
settings.add(PROXY_PORT);
settings.add(PROXY_SCHEME);
settings.add(MAX_HTTP_RESPONSE_SIZE);
settings.add(HOSTS_WHITELIST);
return settings;
}

Expand Down
4 changes: 3 additions & 1 deletion ...her/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java
View file
Expand Up @@ -47,6 +47,7 @@

import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.xpack.watcher.common.http.HttpClientTests.mockClusterService;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.Matchers.containsString;
Expand Down Expand Up @@ -214,7 +215,8 @@ private WebhookActionFactory webhookFactory(HttpClient client) {
public void testThatSelectingProxyWorks() throws Exception {
Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());

try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null);
try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null,
mockClusterService());
MockWebServer proxyServer = new MockWebServer()) {
proxyServer.start();
proxyServer.enqueue(new MockResponse().setResponseCode(200).setBody("fullProxiedContent"));
Expand Down

0 comments on commit bbd0930

Please sign in to comment.