From 34de1e8cbcac714ebaee4b8e269ea081197ef3e4 Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Mon, 1 Jun 2015 01:24:33 +0200 Subject: [PATCH] gzip compression will perform more efficient and with better compression level --- .../net/yacy/cora/protocol/http/GzipCompressingEntity.java | 3 ++- source/net/yacy/kelondro/blob/Compressor.java | 3 ++- source/net/yacy/kelondro/index/RowHandleMap.java | 3 ++- source/net/yacy/search/Switchboard.java | 4 ++-- source/net/yacy/search/index/Fulltext.java | 3 ++- source/net/yacy/utils/gzip.java | 7 ++++--- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java b/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java index 19e4bdfa0e..01c78337ea 100644 --- a/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java +++ b/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.util.zip.Deflater; import java.util.zip.GZIPOutputStream; import org.apache.http.Header; @@ -65,7 +66,7 @@ public void writeTo(final OutputStream outstream) throws IOException { if (outstream == null) { throw new IllegalArgumentException("Output stream may not be null"); } - GZIPOutputStream gzip = new GZIPOutputStream(outstream); + GZIPOutputStream gzip = new GZIPOutputStream(outstream, 65536){{def.setLevel(Deflater.BEST_SPEED);}}; wrappedEntity.writeTo(gzip); gzip.finish(); } diff --git a/source/net/yacy/kelondro/blob/Compressor.java b/source/net/yacy/kelondro/blob/Compressor.java index 8da99e5a90..73fc52b059 100644 --- a/source/net/yacy/kelondro/blob/Compressor.java +++ b/source/net/yacy/kelondro/blob/Compressor.java @@ -35,6 +35,7 @@ import java.util.Iterator; import java.util.Map; import java.util.TreeMap; +import java.util.zip.Deflater; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -114,7 +115,7 @@ private static byte[] compressAddMagic(final byte[] b) { //System.out.print("/(" + cdr + ")"); // DEBUG final ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length / 5); baos.write(gzipMagic); - final OutputStream os = new GZIPOutputStream(baos, 512); + final OutputStream os = new GZIPOutputStream(baos, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}}; os.write(b); os.close(); baos.close(); diff --git a/source/net/yacy/kelondro/index/RowHandleMap.java b/source/net/yacy/kelondro/index/RowHandleMap.java index 9689bdc1ee..3a29c77354 100644 --- a/source/net/yacy/kelondro/index/RowHandleMap.java +++ b/source/net/yacy/kelondro/index/RowHandleMap.java @@ -45,6 +45,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; +import java.util.zip.Deflater; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -189,7 +190,7 @@ public final int dump(final File file) throws IOException { } catch (final OutOfMemoryError e) { os = new FileOutputStream(tmp); } - if (file.getName().endsWith(".gz")) os = new GZIPOutputStream(os); + if (file.getName().endsWith(".gz")) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}}; int c = 0; while (i.hasNext()) { os.write(i.next().bytes()); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 57540271cd..bae47c53f0 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -76,6 +76,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import java.util.zip.Deflater; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; @@ -1959,8 +1960,7 @@ public boolean processSurrogate(final String s) { final String gzname = outfile.getName() + ".gz"; final File gzfile = new File(outfile.getParentFile(), gzname); try { - final OutputStream os = - new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile))); + final OutputStream os = new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}}); BufferedInputStream bis = new BufferedInputStream(new FileInputStream(outfile)); FileUtils.copy(bis, os); os.close(); diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index eb36b44586..130ae4282f 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -41,6 +41,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; +import java.util.zip.Deflater; import java.util.zip.GZIPOutputStream; import net.yacy.cora.date.GenericFormatter; @@ -658,7 +659,7 @@ public void run() { final File parentf = this.f.getParentFile(); if (parentf != null) parentf.mkdirs(); OutputStream os = new FileOutputStream(this.format == 3 ? new File(this.f.getAbsolutePath() + ".gz") : this.f); - if (this.format == 3) os = new GZIPOutputStream(os); + if (this.format == 3) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}}; final PrintWriter pw = new PrintWriter(new BufferedOutputStream(os)); if (this.format == 1) { pw.println(""); diff --git a/source/net/yacy/utils/gzip.java b/source/net/yacy/utils/gzip.java index 2a387cf3b5..c7360bcec4 100644 --- a/source/net/yacy/utils/gzip.java +++ b/source/net/yacy/utils/gzip.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.zip.Deflater; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -49,7 +50,7 @@ public class gzip { public static void gzipFile(final String inFile, final String outFile) { try { final InputStream fin = new BufferedInputStream(new FileInputStream(inFile)); - final OutputStream fout = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(outFile), 1024)); + final OutputStream fout = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(outFile)), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}}; copy(fout, fin, 1024); fin.close(); fout.close(); @@ -87,7 +88,7 @@ public static byte[] gzipString(final String in) { final InputStream fin = new ByteArrayInputStream(in.getBytes("UTF8")); final int buffersize = Math.min(1024, in.length()); final ByteArrayOutputStream baos = new ByteArrayOutputStream(buffersize); - final OutputStream fout = new GZIPOutputStream(baos, Math.max(buffersize, 512)); + final OutputStream fout = new GZIPOutputStream(baos, Math.max(buffersize, 65536)){{def.setLevel(Deflater.BEST_COMPRESSION);}}; copy(fout, fin, 1024); fin.close(); fout.close(); @@ -123,7 +124,7 @@ public static void saveGzip(final File f, final byte[] content) throws IOExcepti java.util.zip.GZIPOutputStream gzipout = null; try { f.getParentFile().mkdirs(); - gzipout = new java.util.zip.GZIPOutputStream(new FileOutputStream(f)); + gzipout = new java.util.zip.GZIPOutputStream(new FileOutputStream(f), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}}; gzipout.write(content, 0, content.length); } finally { if (gzipout!=null)try{gzipout.close();}catch(final Exception e){}