Tweak block selection.

flate/huffman_bit_writer.go

@@ -808,7 +808,8 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
 	}
 
 	// Add everything as literals
-	estBits := histogramSize(input, w.literalFreq[:], !eof && !sync) + 15 + w.lastHeader
+	const guessHeaderSizeBits = 400
+	estBits := histogramSize(input, w.literalFreq[:], !eof && !sync) + 15 + guessHeaderSizeBits
 
 	// Store bytes, if we don't get a reasonable improvement.
 	ssize, storable := w.storedSize(input)
@@ -819,8 +820,10 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
 	}
 
 	if w.lastHeader > 0 {
-		size := w.dynamicReuseSize(w.literalEncoding, huffOffset)
-		estBits += estBits >> w.logReusePenalty
+		size := w.dynamicReuseSize(w.literalEncoding, huffOffset) + w.lastHeader
+		if w.logReusePenalty > 0 {
+			estBits += estBits >> w.logReusePenalty
+		}
 
 		if estBits < size {
 			// We owe an EOB

flate/huffman_bit_writer_test.go

@@ -45,6 +45,7 @@ func testBlockHuff(t *testing.T, in, out string) {
 	}
 	var buf bytes.Buffer
 	bw := newHuffmanBitWriter(&buf)
+	bw.logReusePenalty = 8
 	bw.writeBlockHuff(false, all, false)
 	bw.flush()
 	got := buf.Bytes()

flate/huffman_code.go

@@ -356,6 +356,13 @@ func (s byFreq) Less(i, j int) bool {
 
 func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
 
+func atLeastOne(v float32) float32 {
+	if v < 1 {
+		return 1
+	}
+	return v
+}
+
 // histogramSize accumulates a histogram of b in h.
 // An estimated size in bits is returned.
 // Unassigned values are assigned '1' in the histogram.
@@ -365,15 +372,13 @@ func histogramSize(b []byte, h []uint16, fill bool) int {
 	for _, t := range b {
 		h[t]++
 	}
-	invTotal := float32(1.0 / float64(len(b)))
+	invTotal := 1.0 / float32(len(b))
 	shannon := float32(0.0)
-	single := float32(math.Ceil(-math.Log2(float64(invTotal))))
 	for i, v := range h[:] {
 		if v > 0 {
 			n := float32(v)
-			shannon += -mFastLog2(n*invTotal) * n
+			shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
 		} else if fill {
-			shannon += single
 			h[i] = 1
 		}
 	}

flate/token.go

@@ -184,9 +184,7 @@ func (t *tokens) indexTokens(in []token) {
 	t.Reset()
 	for _, tok := range in {
 		if tok < matchType {
-			t.tokens[t.n] = tok
-			t.litHist[tok]++
-			t.n++
+			t.AddLiteral(tok.literal())
 			continue
 		}
 		t.AddMatch(uint32(tok.length()), tok.offset())
@@ -258,7 +256,6 @@ func (t *tokens) EstimatedBits() int {
 			}
 		}
 	}
-
 	return int(shannon) + bits
 }
 

flate/token_test.go

@@ -1,6 +1,7 @@
 package flate
 
 import (
+	"bytes"
 	"io/ioutil"
 	"testing"
 )
@@ -27,8 +28,17 @@ func loadTestTokens(t testFatal) *tokens {
 func Test_tokens_EstimatedBits(t *testing.T) {
 	tok := loadTestTokens(t)
 	// The estimated size, update if method changes.
-	const expect = 196635
-	if n := tok.EstimatedBits(); n != expect {
+	const expect = 221057
+	n := tok.EstimatedBits()
+	var buf bytes.Buffer
+	wr := newHuffmanBitWriter(&buf)
+	wr.writeBlockDynamic(tok, true, nil, true)
+	if wr.err != nil {
+		t.Fatal(wr.err)
+	}
+	wr.flush()
+	t.Log("got:", n, "actual:", buf.Len()*8, "(header not part of estimate)")
+	if n != expect {
 		t.Error("want:", expect, "bits, got:", n)
 	}
 }