-
Notifications
You must be signed in to change notification settings - Fork 746
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
10 changed files
with
483 additions
and
50 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
142 changes: 142 additions & 0 deletions
142
gobblin-core/src/main/java/gobblin/http/DefaultHttpClientConfigurator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| /* | ||
| * Copyright (C) 2014-2016 LinkedIn Corp. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not use | ||
| * this file except in compliance with the License. You may obtain a copy of the | ||
| * License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software distributed | ||
| * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR | ||
| * CONDITIONS OF ANY KIND, either express or implied. | ||
| */ | ||
| package gobblin.http; | ||
|
|
||
| import java.util.regex.Matcher; | ||
| import java.util.regex.Pattern; | ||
|
|
||
| import org.apache.http.HttpHost; | ||
| import org.apache.http.client.HttpClient; | ||
| import org.apache.http.impl.client.HttpClientBuilder; | ||
|
|
||
| import com.google.common.annotations.VisibleForTesting; | ||
| import com.google.common.base.Optional; | ||
| import com.google.common.base.Strings; | ||
| import com.typesafe.config.Config; | ||
| import com.typesafe.config.ConfigFactory; | ||
| import com.typesafe.config.ConfigValueFactory; | ||
|
|
||
| import gobblin.annotation.Alias; | ||
| import gobblin.configuration.State; | ||
|
|
||
| /** | ||
| * Default implementation that uses the following properties to configure an {@link HttpClient}. | ||
| * | ||
| * <ul> | ||
| * <li>{@link #PROXY_HOSTPORT_KEY} | ||
| * <li>{@link #PROXY_URL_KEY} | ||
| * <li>{@link #PROXY_PORT_KEY} | ||
| * </ul> | ||
| */ | ||
| @Alias(value="default") | ||
| public class DefaultHttpClientConfigurator implements HttpClientConfigurator { | ||
| // IMPORTANT: don't change the values for PROXY_URL_KEY and PROXY_PORT_KEY as they are meant to | ||
| // be backwards compatible with SOURCE_CONN_USE_PROXY_URL and SOURCE_CONN_USE_PROXY_PORT when | ||
| // the statePropertiesPrefix is "source.conn." | ||
| /** The hostname of the HTTP proxy to use */ | ||
| public static final String PROXY_URL_KEY = "use.proxy.url"; | ||
| /** The port of the HTTP proxy to use */ | ||
| public static final String PROXY_PORT_KEY = "use.proxy.port"; | ||
| /** Similar to {@link #PROXY_URL_KEY} and {@link #PROXY_PORT_KEY} but allows you to set it on | ||
| * one property as <host>:<port> */ | ||
| public static final String PROXY_HOSTPORT_KEY = "proxyHostport"; | ||
| /** Port to use if the HTTP Proxy is enabled but no port is specified */ | ||
| public static final int DEFAULT_HTTP_PROXY_PORT = 8080; | ||
|
|
||
| private static final Pattern HOSTPORT_PATTERN = Pattern.compile("([^:]+)(:([0-9]+))?"); | ||
|
|
||
| protected final HttpClientBuilder _builder = HttpClientBuilder.create(); | ||
| protected String _statePropertiesPrefix = null; | ||
|
|
||
| /** {@inheritDoc} */ | ||
| @Override | ||
| public DefaultHttpClientConfigurator configure(Config httpClientConfig) { | ||
| Optional<HttpHost> proxy = getProxyAddr(httpClientConfig); | ||
| if (proxy.isPresent()) { | ||
| getBuilder().setProxy(proxy.get()); | ||
| } | ||
| return this; | ||
| } | ||
|
|
||
| /** {@inheritDoc} */ | ||
| @Override | ||
| public DefaultHttpClientConfigurator configure(State state) { | ||
| Config cfg = stateToConfig(state); | ||
| return configure(cfg); | ||
| } | ||
|
|
||
| protected Config stateToConfig(State state) { | ||
| String proxyUrlKey = getPrefixedPropertyName(PROXY_URL_KEY); | ||
| String proxyPortKey = getPrefixedPropertyName(PROXY_PORT_KEY); | ||
| String proxyHostportKey = getPrefixedPropertyName(PROXY_HOSTPORT_KEY); | ||
|
|
||
| Config cfg = ConfigFactory.empty(); | ||
| if (state.contains(proxyUrlKey)) { | ||
| cfg = cfg.withValue(PROXY_URL_KEY, ConfigValueFactory.fromAnyRef(state.getProp(proxyUrlKey))); | ||
| } | ||
| if (state.contains(proxyPortKey)) { | ||
| cfg = cfg.withValue(PROXY_PORT_KEY, ConfigValueFactory.fromAnyRef(state.getPropAsInt(proxyPortKey))); | ||
| } | ||
| if (state.contains(proxyHostportKey)) { | ||
| cfg = cfg.withValue(PROXY_HOSTPORT_KEY, ConfigValueFactory.fromAnyRef(state.getProp(proxyHostportKey))); | ||
| } | ||
| return cfg; | ||
| } | ||
|
|
||
| /** {@inheritDoc} */ | ||
| @Override | ||
| public HttpClient createClient() { | ||
| return _builder.build(); | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
| public static Optional<HttpHost> getProxyAddr(Config httpClientConfig) { | ||
| String proxyHost = null; | ||
| int proxyPort = DEFAULT_HTTP_PROXY_PORT; | ||
| if (httpClientConfig.hasPath(PROXY_URL_KEY) && | ||
| !httpClientConfig.getString(PROXY_URL_KEY).isEmpty()) { | ||
| proxyHost = httpClientConfig.getString(PROXY_URL_KEY); | ||
| } | ||
| if (httpClientConfig.hasPath(PROXY_PORT_KEY)) { | ||
| proxyPort = httpClientConfig.getInt(PROXY_PORT_KEY); | ||
| } | ||
| if (httpClientConfig.hasPath(PROXY_HOSTPORT_KEY)) { | ||
| String hostport = httpClientConfig.getString(PROXY_HOSTPORT_KEY); | ||
| Matcher hostportMatcher = HOSTPORT_PATTERN.matcher(hostport); | ||
| if (!hostportMatcher.matches()) { | ||
| throw new IllegalArgumentException("Invalid HTTP proxy hostport: " + hostport); | ||
| } | ||
| proxyHost = hostportMatcher.group(1); | ||
| if (!Strings.isNullOrEmpty(hostportMatcher.group(3))) { | ||
| proxyPort = Integer.parseInt(hostportMatcher.group(3)); | ||
| } | ||
| } | ||
| return null != proxyHost ? Optional.of(new HttpHost(proxyHost, proxyPort)) | ||
| : Optional.<HttpHost>absent(); | ||
| } | ||
|
|
||
| @Override | ||
| public DefaultHttpClientConfigurator setStatePropertiesPrefix(String propertiesPrefix) { | ||
| _statePropertiesPrefix = propertiesPrefix; | ||
| return this; | ||
| } | ||
|
|
||
| String getPrefixedPropertyName(String propertyName) { | ||
| return null != _statePropertiesPrefix ? _statePropertiesPrefix + propertyName : propertyName; | ||
| } | ||
|
|
||
| @Override | ||
| public HttpClientBuilder getBuilder() { | ||
| return _builder; | ||
| } | ||
|
|
||
| } |
48 changes: 48 additions & 0 deletions
48
gobblin-core/src/main/java/gobblin/http/HttpClientConfigurator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| /* | ||
| * Copyright (C) 2014-2016 LinkedIn Corp. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not use | ||
| * this file except in compliance with the License. You may obtain a copy of the | ||
| * License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software distributed | ||
| * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR | ||
| * CONDITIONS OF ANY KIND, either express or implied. | ||
| */ | ||
| package gobblin.http; | ||
|
|
||
| import org.apache.http.client.HttpClient; | ||
| import org.apache.http.impl.client.HttpClientBuilder; | ||
|
|
||
| import com.typesafe.config.Config; | ||
|
|
||
| import gobblin.configuration.State; | ||
|
|
||
| /** | ||
| * An adapter from Gobblin configuration to {@link HttpClientBuilder}. It can also be used to | ||
| * create {@link HttpClient} instances. | ||
| */ | ||
| public interface HttpClientConfigurator { | ||
|
|
||
| /** Sets a prefix to use when extracting the configuration from {@link State}. The default is | ||
| * empty. */ | ||
| HttpClientConfigurator setStatePropertiesPrefix(String propertiesPrefix); | ||
|
|
||
| /** | ||
| * Extracts the HttpClient configuration from a typesafe config. Supported configuration options | ||
| * may vary from implementation to implementation. | ||
| * */ | ||
| HttpClientConfigurator configure(Config httpClientConfig); | ||
|
|
||
| /** Same as {@link #configure(Config)} but for legacy cases using State. */ | ||
| HttpClientConfigurator configure(State httpClientConfig); | ||
|
|
||
| /** The underlying client builder */ | ||
| HttpClientBuilder getBuilder(); | ||
|
|
||
| /** | ||
| * Typically this will use {@link HttpClientBuilder#build()} based on the configuration but | ||
| * implementations may also return decorated instances. */ | ||
| HttpClient createClient(); | ||
|
|
||
| } |
74 changes: 74 additions & 0 deletions
74
gobblin-core/src/main/java/gobblin/http/HttpClientConfiguratorLoader.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| /* | ||
| * Copyright (C) 2014-2016 LinkedIn Corp. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not use | ||
| * this file except in compliance with the License. You may obtain a copy of the | ||
| * License at http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software distributed | ||
| * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR | ||
| * CONDITIONS OF ANY KIND, either express or implied. | ||
| */ | ||
| package gobblin.http; | ||
|
|
||
| import org.apache.http.client.HttpClient; | ||
|
|
||
| import com.google.common.base.Optional; | ||
| import com.typesafe.config.Config; | ||
|
|
||
| import gobblin.configuration.State; | ||
| import gobblin.util.ClassAliasResolver; | ||
|
|
||
| /** | ||
| * Creates an instance of HttpClientConfigurator using dependency injection from configuration. | ||
| */ | ||
| public class HttpClientConfiguratorLoader { | ||
|
|
||
| /** Classname or alias for an {@link HttpClientConfigurator} instance to use for configuring and | ||
| * instantiating of {@link HttpClient} instances. */ | ||
| public static final String HTTP_CLIENT_CONFIGURATOR_TYPE_KEY = "httpClientConfigurator.type"; | ||
| public static final String HTTP_CLIENT_CONFIGURATOR_TYPE_FULL_KEY = | ||
| "gobblin." + HTTP_CLIENT_CONFIGURATOR_TYPE_KEY; | ||
| public static final Class<? extends HttpClientConfigurator> DEFAULT_CONFIGURATOR_CLASS = | ||
| DefaultHttpClientConfigurator.class; | ||
|
|
||
| private static final ClassAliasResolver<HttpClientConfigurator> TYPE_RESOLVER = | ||
| new ClassAliasResolver<>(HttpClientConfigurator.class); | ||
| private final HttpClientConfigurator _configurator; | ||
|
|
||
| /** | ||
| * Loads a HttpClientConfigurator using the value of the {@link #HTTP_CLIENT_CONFIGURATOR_TYPE_FULL_KEY} | ||
| * property in the state. | ||
| */ | ||
| public HttpClientConfiguratorLoader(State state) { | ||
| this(Optional.<String>fromNullable(state.getProp(HTTP_CLIENT_CONFIGURATOR_TYPE_FULL_KEY))); | ||
| } | ||
|
|
||
| /** Loads a HttpClientConfigurator using the value of {@link #HTTP_CLIENT_CONFIGURATOR_TYPE_KEY} | ||
| * in the local typesafe config. */ | ||
| public HttpClientConfiguratorLoader(Config config) { | ||
| this(Optional.<String>fromNullable(config.hasPath(HTTP_CLIENT_CONFIGURATOR_TYPE_KEY) ? | ||
| config.getString(HTTP_CLIENT_CONFIGURATOR_TYPE_KEY) : null)); | ||
| } | ||
|
|
||
| /** Loads a HttpClientConfigurator with the specified class or alias. If not specified, | ||
| * {@link #DEFAULT_CONFIGURATOR_CLASS} is used. */ | ||
| public HttpClientConfiguratorLoader(Optional<String> configuratorType) { | ||
| try { | ||
| _configurator = getConfiguratorClass(configuratorType).newInstance(); | ||
| } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { | ||
| throw new RuntimeException("Unable to find HttpClientConfigurator:" + e, e); | ||
| } | ||
| } | ||
|
|
||
| private static Class<? extends HttpClientConfigurator> | ||
| getConfiguratorClass(Optional<String> configuratorType) throws ClassNotFoundException { | ||
| return configuratorType.isPresent() ? TYPE_RESOLVER.resolveClass(configuratorType.get()) : | ||
| DEFAULT_CONFIGURATOR_CLASS; | ||
| } | ||
|
|
||
| public HttpClientConfigurator getConfigurator() { | ||
| return _configurator; | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.