Skip to content
Permalink
Browse files

Fixed gzip decompression regression on index transfer APIs

Processing of gzip encoded incoming requests (on /yacy/transferRWI.html
and /yacy/transferURL.html) was no more working since upgrade to Jetty
9.4.12 (see commit 51f4be1).

To prevent any conflicting behavior with Jetty internals, use now the
GzipHandler provided by Jetty to decompress incoming gzip encoded
requests rather than the previously used custom GZIPRequestWrapper.

Fixes issue #249
  • Loading branch information...
luccioman committed Nov 7, 2018
1 parent e85f231 commit a99713326023054769cc2fade536b59743a2d7cb
Showing with 31 additions and 164 deletions.
  1. +13 −10 source/net/yacy/http/Jetty9HttpServerImpl.java
  2. +18 −154 source/net/yacy/http/servlets/YaCyDefaultServlet.java
@@ -34,6 +34,7 @@
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;

import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Handler;
@@ -144,18 +145,20 @@ public Jetty9HttpServerImpl(int port) {
//sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html
htrootContext.addServlet(sholder, "/*");

/* Handle gzip compression of responses to user agents accepting it */
final GzipHandler gzipHandler;
if (sb.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
final GzipHandler gzipHandler = new GzipHandler();
/*
* Decompression of incoming requests body is required for index distribution
* APIs /yacy/transferRWI.html and /yacy/transferURL.html This was previously
* handled by a GZIPRequestWrapper in the YaCyDefaultServlet.
*/
gzipHandler.setInflateBufferSize(4096);

if (!sb.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT)) {
gzipHandler = new GzipHandler();
/*
* Ensure decompression of requests body is disabled : it is already handled by
* the GZIPRequestWrapper in the YaCyDefaultServlet
*/
gzipHandler.setInflateBufferSize(0);
htrootContext.setGzipHandler(gzipHandler);
/* Gzip compression of responses can be disabled by user configuration */
gzipHandler.setExcludedMethods(HttpMethod.GET.asString(), HttpMethod.POST.asString());
}
htrootContext.setGzipHandler(gzipHandler);

// -----------------------------------------------------------------------------
// here we set and map the mandatory servlets, needed for typical YaCy operation
@@ -42,25 +42,33 @@
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;

import javax.servlet.ReadListener;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
import javax.servlet.UnavailableException;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.MimeTypes;
import org.eclipse.jetty.io.WriterOutputStream;
import org.eclipse.jetty.server.InclusiveByteRange;
import org.eclipse.jetty.util.MultiPartOutputStream;
import org.eclipse.jetty.util.URIUtil;
import org.eclipse.jetty.util.resource.Resource;

import com.google.common.net.HttpHeaders;

import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.order.Base64Order;
@@ -75,38 +83,19 @@
import net.yacy.data.TransactionManager;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;
import net.yacy.kelondro.util.NamePrefixThreadFactory;
import net.yacy.peers.Seed;
import net.yacy.peers.graphics.EncodedImage;
import net.yacy.peers.operation.yacyBuildProperties;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
import net.yacy.server.http.HTTPDFileHandler;
import net.yacy.server.http.TemplateEngine;
import net.yacy.server.serverClassLoader;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
import net.yacy.server.servletProperties;
import net.yacy.server.http.HTTPDFileHandler;
import net.yacy.server.http.TemplateEngine;
import net.yacy.visualization.RasterPlotter;

import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.MimeTypes;
import org.eclipse.jetty.io.WriterOutputStream;
import org.eclipse.jetty.server.InclusiveByteRange;
import org.eclipse.jetty.util.MultiPartOutputStream;
import org.eclipse.jetty.util.URIUtil;
import org.eclipse.jetty.util.resource.Resource;

import com.google.common.net.HttpHeaders;
import com.google.common.util.concurrent.SimpleTimeLimiter;
import com.google.common.util.concurrent.TimeLimiter;
import com.google.common.util.concurrent.UncheckedTimeoutException;

/**
* YaCyDefaultServlet based on Jetty DefaultServlet.java
* handles static files and the YaCy servlets.
@@ -152,8 +141,6 @@
protected static final File TMPDIR = new File(System.getProperty("java.io.tmpdir"));
protected static final int SIZE_FILE_THRESHOLD = 1024 * 1024 * 1024; // 1GB is a lot but appropriate for multi-document pushed using the push_p.json servlet
protected static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(SIZE_FILE_THRESHOLD, TMPDIR);
private final static TimeLimiter timeLimiter = new SimpleTimeLimiter(Executors.newCachedThreadPool(
new NamePrefixThreadFactory(YaCyDefaultServlet.class.getSimpleName() + ".timeLimiter")));
/* ------------------------------------------------------------ */
@Override
public void init() throws UnavailableException {
@@ -866,12 +853,7 @@ protected void handleTemplate(String target, HttpServletRequest request, HttpSe
RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt);
// add multipart-form fields to parameter
if (ServletFileUpload.isMultipartContent(request)) {
final String bodyEncoding = request.getHeader(HeaderFramework.CONTENT_ENCODING);
if (HeaderFramework.CONTENT_ENCODING_GZIP.equalsIgnoreCase(bodyEncoding)) {
parseMultipart(new GZIPRequestWrapper(request),args);
} else {
parseMultipart(request, args);
}
parseMultipart(request, args);
}
// eof modification to read attribute
Object tmp;
@@ -1336,122 +1318,4 @@ public void run() {
ConcurrentLog.info("FILEHANDLER", ex.getMessage());
}
}

/**
* wraps request to uncompress gzip'ed input stream
*/
private class GZIPRequestWrapper extends HttpServletRequestWrapper {

private final ServletInputStream is;

public GZIPRequestWrapper(HttpServletRequest request) throws IOException {
super(request);
this.is = new GZIPRequestStream(request);
}

@Override
public ServletInputStream getInputStream() throws IOException {
return is;
}

}

private class GZIPRequestStream extends ServletInputStream {

private final GZIPInputStream in;
private final ServletInputStream sin;

public GZIPRequestStream(HttpServletRequest request) throws IOException {
sin = request.getInputStream();
in = new GZIPInputStream(sin);
}

@Override
public int read() throws IOException {
return in.read();
}

@Override
public int read(byte[] b) throws IOException {
return read(b, 0, b.length);
}

@Override
public int read(byte[] b, int off, int len) throws IOException {
try {
return timeLimiter.callWithTimeout(new CallableReader(in, b, off, len), len + 600, TimeUnit.MILLISECONDS, false);
} catch (final UncheckedTimeoutException e) {
return -1;
} catch (Exception e) {
throw new IOException(e);
}
}

@Override
public void close() throws IOException {
in.close();
}

@Override
public int available() throws IOException {
return in.available();
}

@Override
public synchronized void mark(int readlimit) {
in.mark(readlimit);
}

@Override
public boolean markSupported() {
return in.markSupported();
}

@Override
public synchronized void reset() throws IOException {
in.reset();
}

@Override
public long skip(long n) throws IOException {
return in.skip(n);
}

@Override
public boolean isFinished() {
try {
return available() < 1;
} catch (final IOException ex) {
return true;
}
}

@Override
public boolean isReady() {
return sin.isReady() && !isFinished();
}

@Override
public void setReadListener(ReadListener rl) {
sin.setReadListener(rl);
}
}

private class CallableReader implements Callable<Integer> {
private int off, len;
private byte[] b;
private GZIPInputStream in;

public CallableReader(final GZIPInputStream in, byte[] b, int off, int len) {
this.in = in;
this.b = b;
this.off = off;
this.len = len;
}

@Override
public Integer call() throws Exception {
return in.read(b, off, len);
}
}
}

0 comments on commit a997133

Please sign in to comment.
You can’t perform that action at this time.