Skip to content

Commit

Permalink
push.go: push metrics to remote storage in gzip-compressed form
Browse files Browse the repository at this point in the history
This should reduce the needed network bandwidth by ~10x for reading the metrics at remote storage side
  • Loading branch information
valyala committed Jul 25, 2022
1 parent f790ba5 commit b305bc0
Showing 1 changed file with 22 additions and 2 deletions.
24 changes: 22 additions & 2 deletions push.go
Expand Up @@ -9,6 +9,8 @@ import (
"net/http"
"net/url"
"time"

"compress/gzip"
)

// InitPushProcessMetrics sets up periodic push for 'process_*' metrics to the given pushURL with the given interval.
Expand Down Expand Up @@ -86,6 +88,9 @@ func (s *Set) InitPush(pushURL string, interval time.Duration, extraLabels strin
//
// It is OK calling InitPushExt multiple times with different pushURL -
// in this case metrics are pushed to all the provided pushURL urls.
//
// It is OK calling InitPushExt multiple times with different writeMetrics -
// in this case all the metrics generated by writeMetrics callbacks are writte to pushURL.
func InitPushExt(pushURL string, interval time.Duration, extraLabels string, writeMetrics func(w io.Writer)) error {
if interval <= 0 {
return fmt.Errorf("interval must be positive; got %s", interval)
Expand All @@ -111,15 +116,30 @@ func InitPushExt(pushURL string, interval time.Duration, extraLabels string, wri
ticker := time.NewTicker(interval)
var bb bytes.Buffer
var tmpBuf []byte
zw := gzip.NewWriter(&bb)
for range ticker.C {
bb.Reset()
writeMetrics(&bb)
if len(extraLabels) > 0 {
tmpBuf = addExtraLabels(tmpBuf[:0], bb.Bytes(), extraLabels)
bb.Reset()
bb.Write(tmpBuf)
if _, err := bb.Write(tmpBuf); err != nil {
panic(fmt.Errorf("BUG: cannot write %d bytes to bytes.Buffer: %s", len(tmpBuf), err))
}
}
tmpBuf = append(tmpBuf[:0], bb.Bytes()...)
bb.Reset()
zw.Reset(&bb)
if _, err := zw.Write(tmpBuf); err != nil {
panic(fmt.Errorf("BUG: cannot write %d bytes to gzip writer: %s", len(tmpBuf), err))
}
req, err := http.NewRequest("GET", pushURL, &bb)
if err != nil {
log.Printf("ERROR: metrics.push: cannot initialize request for metrics push to %q: %s", pushURLRedacted, err)
}
resp, err := c.Post(pushURL, "text/plain", &bb)
req.Header.Set("Content-Type", "text/plain")
req.Header.Set("Content-Encoding", "gzip")
resp, err := c.Do(req)
if err != nil {
log.Printf("ERROR: metrics.push: cannot push metrics to %q: %s", pushURLRedacted, err)
continue
Expand Down

0 comments on commit b305bc0

Please sign in to comment.