Permalink
Browse files

Support Brotli encoding in http_request

Also allow decompression to be skipped in the options.
Speed up provisional-build by skipping code coverage calculations.
  • Loading branch information...
LadyCailin committed Dec 20, 2018
1 parent 082e0c0 commit 25a2655582a2f94392bf8ea82a9acaee9c42e548
33 pom.xml
@@ -302,13 +302,19 @@
<artifactId>jchardet</artifactId>
<version>1.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.oltu.oauth2/org.apache.oltu.oauth2.client -->
<dependency>
<!-- Apache License 2.0 -->
<groupId>org.apache.oltu.oauth2</groupId>
<artifactId>org.apache.oltu.oauth2.client</artifactId>
<version>1.0.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.oltu.oauth2/org.apache.oltu.oauth2.client -->
<dependency>
<!-- Apache License 2.0 -->
<groupId>org.apache.oltu.oauth2</groupId>
<artifactId>org.apache.oltu.oauth2.client</artifactId>
<version>1.0.2</version>
</dependency>
<dependency>
<!-- MIT -->
<groupId>org.brotli</groupId>
<artifactId>dec</artifactId>
<version>0.1.2</version>
</dependency>
<!-- Test libraries -->
<!-- Hamcrest must come before powermock -->
<dependency>
@@ -514,6 +520,7 @@
<include>javax.activation:activation:jar:*</include>
<include>postgresql:postgresql:jar:*</include>
<include>org.apache.oltu.oauth2:org.apache.oltu.oauth2.client:jar:*</include>
<include>org.brotli:dec:jar:*</include>
</includes>
</artifactSet>
<relocations>
@@ -613,6 +620,10 @@
<pattern>org.apache.oltu.oauth2</pattern>
<shadedPattern>com.laytonsmith.libs.org.apache.oltu.oauth2</shadedPattern>
</relocation>
<relocation>
<pattern>org.brotli</pattern>
<shadedPattern>com.laytonsmith.libs.org.brotli</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
@@ -912,6 +923,14 @@
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<version>0.8.2</version>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</profile>
@@ -33,6 +33,7 @@
private LinkedHashMap<String, String> trustStore = new LinkedHashMap<>();
@SuppressWarnings("NonConstantLogger")
private Logger logger;
private boolean disableDecompressionHandling = false;

/**
*
@@ -374,4 +375,23 @@ public RequestSettings setTrustStore(LinkedHashMap<String, String> trustStore) {
return new LinkedHashMap<>(trustStore);
}

/**
* Sets the disableCompressionHandling flag. If true, the content will be returned as is, no matter what the
* value of the Content-Encoding header is, and must be processed manually.
* @param disableCompressionHandling
* @return
*/
public RequestSettings setDisableCompressionHandling(boolean disableCompressionHandling) {
this.disableDecompressionHandling = disableCompressionHandling;
return this;
}

/**
* Returns the disableCompressionHandling flag. Defaults to false.
* @return
*/
public boolean getDisableCompressionHandling() {
return this.disableDecompressionHandling;
}

}
@@ -29,26 +29,34 @@
import java.security.cert.CertificateEncodingException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
import java.util.zip.InflaterInputStream;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLHandshakeException;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import org.apache.commons.codec.binary.Base64;
import org.brotli.dec.BrotliInputStream;

/**
* Contains methods to simplify web connections.
@@ -63,6 +71,15 @@ public static void main(String[] args) throws Exception {
StreamUtils.GetSystemOut().println(stash.getCookies(new URL("http://www.google.com")));
}

/**
* This is the list of encodings that this class supports. Generally speaking, this is the list that you should
* provide in the Accept-Encoding list. If you wish to support something other than this list, you should
* disable encoding support, and manage the decompression entirely yourself.
*/
public static final Set<String> SUPPORTED_ENCODINGS
= Collections.unmodifiableSet(new HashSet<>(
Arrays.asList(new String[]{"gzip", "deflate", "br", "identity"})));

private WebUtility() {
}
private static int urlRetrieverPoolId = 0;
@@ -487,26 +504,49 @@ public Socket createSocket(InetAddress ia, int i, InetAddress ia1, int i1) throw
if(logger != null) {
logger.log(Level.SEVERE, "Exception occurred, {0} response from server", conn.getResponseCode());
}
if(e instanceof SSLHandshakeException) {
// The certificate was not valid, and the input stream will be null anyways, so just throw at this
// point.
throw new IOException("Invalid SSL certificate for " + url.getHost() + ". Refusing to connect.");
}
is = conn.getErrorStream();
}
if("x-gzip".equals(conn.getContentEncoding()) || "gzip".equals(conn.getContentEncoding())) {
if(logger != null) {
logger.log(Level.INFO, "Response is gzipped, using a GZIPInputStream");
}
is = new GZIPInputStream(is);
} else if("deflate".equals(conn.getContentEncoding())) {
if(logger != null) {
logger.log(Level.INFO, "Response is zipped, using a InflaterInputStream");
}
is = new InflaterInputStream(is);
} else if("identity".equals(conn.getContentEncoding())) {
//This is the default, meaning no transformation is needed.
if(logger != null) {
logger.log(Level.INFO, "Response is not compressed");

if(settings.getDisableCompressionHandling()) {
/*
The HTTP spec for Content-Encoding specifies that multiple comma separated values can be provided. Where
more than one is provided, this means that the content was compressed multiple times, in the specified order.
Given that, we must loop through the list, wrapping the input stream in the given decompression handlers.
In practice, this will only loop once though.
*/
List<String> compression
= Stream.of(conn.getContentEncoding().split(",")).map((e) -> e.trim()).collect(Collectors.toList());
for(String scheme : compression) {
if("x-gzip".equals(scheme) || "gzip".equals(scheme)) {
if(logger != null) {
logger.log(Level.INFO, "Response is gzipped, using a GZIPInputStream");
}
is = new GZIPInputStream(is);
} else if("deflate".equals(scheme)) {
if(logger != null) {
logger.log(Level.INFO, "Response is zipped, using an InflaterInputStream");
}
is = new InflaterInputStream(is);
} else if("br".equals(scheme)) {
if(logger != null) {
logger.log(Level.INFO, "Response is Brotli compressed, using a BrotliInputStream");
}
is = new BrotliInputStream(is);
} else if("identity".equals(scheme)) {
//This is the default, meaning no transformation is needed.
if(logger != null) {
logger.log(Level.INFO, "Response is not compressed");
}
}
}
}
if(is == null) {
throw new IOException("Could not connnect to " + url);
throw new IOException("Could not connect to " + url);
}
return new RawHTTPResponse(conn, is);
}
@@ -196,7 +196,7 @@ public Thread newThread(Runnable r) {

static {
DEFAULT_HEADERS.put("Accept", "text/*, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8");
DEFAULT_HEADERS.put("Accept-Encoding", "gzip, deflate, identity");
DEFAULT_HEADERS.put("Accept-Encoding", StringUtils.Join(WebUtility.SUPPORTED_ENCODINGS, ", "));
DEFAULT_HEADERS.put("User-Agent", "Java/" + System.getProperty("java.version") + "/" + Implementation.GetServerType().getBranding());
DEFAULT_HEADERS.put("DNT", "1");
DEFAULT_HEADERS.put("Connection", "close");

0 comments on commit 25a2655

Please sign in to comment.