Merge pull request #3267 from weaveworks/pool-gzwriters

 Re-use gzip writers in a pool

app/router.go

@@ -11,7 +11,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/PuerkitoBio/ghost/handlers"
+	"github.com/NYTimes/gziphandler"
 	log "github.com/Sirupsen/logrus"
 	"github.com/gorilla/mux"
 	"github.com/ugorji/go/codec"
@@ -86,32 +86,29 @@ func matchURL(r *http.Request, pattern string) (map[string]string, bool) {
 	return vars, true
 }
 
-func gzipHandler(h http.HandlerFunc) http.HandlerFunc {
-	return handlers.GZIPHandlerFunc(h, nil)
+func gzipHandler(h http.HandlerFunc) http.Handler {
+	return gziphandler.GzipHandler(h)
 }
 
 // RegisterTopologyRoutes registers the various topology routes with a http mux.
 func RegisterTopologyRoutes(router *mux.Router, r Reporter, capabilities map[string]bool) {
 	get := router.Methods("GET").Subrouter()
-	get.HandleFunc("/api",
+	get.Handle("/api",
 		gzipHandler(requestContextDecorator(apiHandler(r, capabilities))))
-	get.HandleFunc("/api/topology",
+	get.Handle("/api/topology",
 		gzipHandler(requestContextDecorator(topologyRegistry.makeTopologyList(r))))
-	get.
-		HandleFunc("/api/topology/{topology}",
-			gzipHandler(requestContextDecorator(topologyRegistry.captureRenderer(r, handleTopology)))).
+	get.Handle("/api/topology/{topology}",
+		gzipHandler(requestContextDecorator(topologyRegistry.captureRenderer(r, handleTopology)))).
 		Name("api_topology_topology")
-	get.
-		HandleFunc("/api/topology/{topology}/ws",
-			requestContextDecorator(captureReporter(r, handleWebsocket))). // NB not gzip!
+	get.Handle("/api/topology/{topology}/ws",
+		requestContextDecorator(captureReporter(r, handleWebsocket))). // NB not gzip!
 		Name("api_topology_topology_ws")
-	get.
-		MatcherFunc(URLMatcher("/api/topology/{topology}/{id}")).HandlerFunc(
+	get.MatcherFunc(URLMatcher("/api/topology/{topology}/{id}")).Handler(
 		gzipHandler(requestContextDecorator(topologyRegistry.captureRenderer(r, handleNode)))).
 		Name("api_topology_topology_id")
-	get.HandleFunc("/api/report",
+	get.Handle("/api/report",
 		gzipHandler(requestContextDecorator(makeRawReportHandler(r))))
-	get.HandleFunc("/api/probes",
+	get.Handle("/api/probes",
 		gzipHandler(requestContextDecorator(makeProbeHandler(r))))
 }
 

extras/copyreport/main.go

@@ -2,7 +2,6 @@
 package main
 
 import (
-	"compress/gzip"
 	"flag"
 	"log"
 
@@ -20,7 +19,7 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
-	if err = rpt.WriteToFile(flag.Arg(1), gzip.DefaultCompression); err != nil {
+	if err = rpt.WriteToFile(flag.Arg(1)); err != nil {
 		log.Fatal(err)
 	}
 }

report/marshal.go

@@ -39,11 +39,10 @@ func (StdoutPublisher) Publish(rep Report) error {
 // WriteBinary writes a Report as a gzipped msgpack into a bytes.Buffer
 func (rep Report) WriteBinary() (*bytes.Buffer, error) {
 	w := &bytes.Buffer{}
-	gzwriter, err := gzip.NewWriterLevel(w, gzip.DefaultCompression)
-	if err != nil {
-		return nil, err
-	}
-	if err = codec.NewEncoder(gzwriter, &codec.MsgpackHandle{}).Encode(&rep); err != nil {
+	gzwriter := gzipWriterPool.Get().(*gzip.Writer)
+	gzwriter.Reset(w)
+	defer gzipWriterPool.Put(gzwriter)
+	if err := codec.NewEncoder(gzwriter, &codec.MsgpackHandle{}).Encode(&rep); err != nil {
 		return nil, err
 	}
 	gzwriter.Close() // otherwise the content won't get flushed to the output stream
@@ -61,10 +60,14 @@ func (c byteCounter) Read(p []byte) (n int, err error) {
 	return n, err
 }
 
-// buffer pool to reduce garbage-collection
+// buffer pools to reduce garbage-collection
 var bufferPool = &sync.Pool{
 	New: func() interface{} { return new(bytes.Buffer) },
 }
+var gzipWriterPool = &sync.Pool{
+	// NewWriterLevel() only errors if the compression level is invalid, which can't happen here
+	New: func() interface{} { w, _ := gzip.NewWriterLevel(nil, gzip.DefaultCompression); return w },
+}
 
 // ReadBinary reads bytes into a Report.
 //
@@ -169,7 +172,7 @@ func MakeFromFile(path string) (rpt Report, _ error) {
 // WriteToFile writes a Report to a file. The encoding is determined
 // by the file extension (".msgpack" or ".json", with an optional
 // ".gz").
-func (rep *Report) WriteToFile(path string, compressionLevel int) error {
+func (rep *Report) WriteToFile(path string) error {
 	f, err := os.Create(path)
 	if err != nil {
 		return err
@@ -186,10 +189,9 @@ func (rep *Report) WriteToFile(path string, compressionLevel int) error {
 	defer bufwriter.Flush()
 	w = bufwriter
 	if gzipped {
-		gzwriter, err := gzip.NewWriterLevel(w, compressionLevel)
-		if err != nil {
-			return err
-		}
+		gzwriter := gzipWriterPool.Get().(*gzip.Writer)
+		gzwriter.Reset(w)
+		defer gzipWriterPool.Put(gzwriter)
 		defer gzwriter.Close()
 		w = gzwriter
 	}