gzip compression will perform more efficient and with better compression

level
yacy · May 31, 2015 · 34de1e8 · 34de1e8
1 parent 98be59c
commit 34de1e8
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 9 deletions.
diff --git a/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java b/source/net/yacy/cora/protocol/http/GzipCompressingEntity.java
@@ -27,6 +27,7 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.zip.Deflater;
 import java.util.zip.GZIPOutputStream;
 
 import org.apache.http.Header;
@@ -65,7 +66,7 @@ public void writeTo(final OutputStream outstream) throws IOException {
 		if (outstream == null) {
 			throw new IllegalArgumentException("Output stream may not be null");
 		}
-		GZIPOutputStream gzip = new GZIPOutputStream(outstream);
+		GZIPOutputStream gzip = new GZIPOutputStream(outstream, 65536){{def.setLevel(Deflater.BEST_SPEED);}};
 		wrappedEntity.writeTo(gzip);
 		gzip.finish();
 	}

diff --git a/source/net/yacy/kelondro/blob/Compressor.java b/source/net/yacy/kelondro/blob/Compressor.java
@@ -35,6 +35,7 @@
 import java.util.Iterator;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.zip.Deflater;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
@@ -114,7 +115,7 @@ private static byte[] compressAddMagic(final byte[] b) {
             //System.out.print("/(" + cdr + ")"); // DEBUG
             final ByteArrayOutputStream baos = new ByteArrayOutputStream(b.length / 5);
             baos.write(gzipMagic);
-            final OutputStream os = new GZIPOutputStream(baos, 512);
+            final OutputStream os = new GZIPOutputStream(baos, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
             os.write(b);
             os.close();
             baos.close();

diff --git a/source/net/yacy/kelondro/index/RowHandleMap.java b/source/net/yacy/kelondro/index/RowHandleMap.java
@@ -45,6 +45,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
+import java.util.zip.Deflater;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
@@ -189,7 +190,7 @@ public final int dump(final File file) throws IOException {
         } catch (final OutOfMemoryError e) {
             os = new FileOutputStream(tmp);
         }
-        if (file.getName().endsWith(".gz")) os = new GZIPOutputStream(os);
+        if (file.getName().endsWith(".gz")) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
         int c = 0;
         while (i.hasNext()) {
             os.write(i.next().bytes());

diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java
@@ -76,6 +76,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
+import java.util.zip.Deflater;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 import java.util.zip.ZipEntry;
@@ -1959,8 +1960,7 @@ public boolean processSurrogate(final String s) {
                         final String gzname = outfile.getName() + ".gz";
                         final File gzfile = new File(outfile.getParentFile(), gzname);
                         try {
-                            final OutputStream os =
-                                new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile)));
+                            final OutputStream os = new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(gzfile), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}});
                             BufferedInputStream bis = new BufferedInputStream(new FileInputStream(outfile)); 
                             FileUtils.copy(bis, os);
                             os.close();

diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java
@@ -41,6 +41,7 @@
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.regex.Pattern;
+import java.util.zip.Deflater;
 import java.util.zip.GZIPOutputStream;
 
 import net.yacy.cora.date.GenericFormatter;
@@ -658,7 +659,7 @@ public void run() {
                 final File parentf = this.f.getParentFile();
                 if (parentf != null) parentf.mkdirs();
                 OutputStream os = new FileOutputStream(this.format == 3 ? new File(this.f.getAbsolutePath() + ".gz") : this.f);
-                if (this.format == 3) os = new GZIPOutputStream(os);
+                if (this.format == 3) os = new GZIPOutputStream(os, 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
                 final PrintWriter pw = new PrintWriter(new BufferedOutputStream(os));
                 if (this.format == 1) {
                     pw.println("<html><head></head><body>");

diff --git a/source/net/yacy/utils/gzip.java b/source/net/yacy/utils/gzip.java
@@ -35,6 +35,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.zip.Deflater;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
@@ -49,7 +50,7 @@ public class gzip {
     public static void gzipFile(final String inFile, final String outFile) {
 	try {
 	    final InputStream  fin  = new BufferedInputStream(new FileInputStream(inFile));
-	    final OutputStream fout = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(outFile), 1024));
+	    final OutputStream fout = new GZIPOutputStream(new BufferedOutputStream(new FileOutputStream(outFile)), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
 	    copy(fout, fin, 1024);
 	    fin.close();
 	    fout.close();
@@ -87,7 +88,7 @@ public static byte[] gzipString(final String in) {
 	    final InputStream fin = new ByteArrayInputStream(in.getBytes("UTF8"));
 	    final int buffersize = Math.min(1024, in.length());
 	    final ByteArrayOutputStream baos = new ByteArrayOutputStream(buffersize);
-	    final OutputStream fout = new GZIPOutputStream(baos, Math.max(buffersize, 512));
+	    final OutputStream fout = new GZIPOutputStream(baos, Math.max(buffersize, 65536)){{def.setLevel(Deflater.BEST_COMPRESSION);}};
 	    copy(fout, fin, 1024);
 	    fin.close();
 	    fout.close();
@@ -123,7 +124,7 @@ public static void saveGzip(final File f, final byte[] content) throws IOExcepti
         java.util.zip.GZIPOutputStream gzipout = null;
         try {
             f.getParentFile().mkdirs();
-            gzipout = new java.util.zip.GZIPOutputStream(new FileOutputStream(f));
+            gzipout = new java.util.zip.GZIPOutputStream(new FileOutputStream(f), 65536){{def.setLevel(Deflater.BEST_COMPRESSION);}};
             gzipout.write(content, 0, content.length);
         } finally {
             if (gzipout!=null)try{gzipout.close();}catch(final Exception e){}