fbclock-sidecar support metrics log sampling

Summary: Reducing the amount of data we log.

Reviewed By: vvfedorenko

Differential Revision: D44256377

fbshipit-source-id: 31b87d0ca91491b6563b516414e9ee0ae78810f8

cmd/fbclock-daemon/main.go

@@ -38,6 +38,7 @@ func main() {
 		manageDevice   bool
 		csvLog         bool
 		csvPath        string
+		logSampleRate  int
 		verbose        bool
 		monitoringPort int
 	)
@@ -63,6 +64,7 @@ func main() {
 	flag.BoolVar(&manageDevice, "manage", true, fmt.Sprintf("Manage device. This will setup %q as a copy of PHC device associated with given network interface", daemon.ManagedPTPDevicePath))
 	flag.BoolVar(&csvLog, "csvlog", true, "Log all the metrics as CSV to log")
 	flag.StringVar(&csvPath, "csvpath", "", "write CSV log into this file")
+	flag.IntVar(&logSampleRate, "logsamplerate", 1, "Sample metrics logs at this rate. 0 means metrics logging is turned off. 1 means every sample is logged, 100 means roughly one in 100 samples will be logged")
 	flag.BoolVar(&verbose, "verbose", false, "Verbose logging")
 
 	flag.Parse()
@@ -94,7 +96,7 @@ func main() {
 	// set up sample logging
 	w := log.StandardLogger().Writer()
 	defer w.Close()
-	var l daemon.Logger = daemon.NewDummyLogger(w)
+	var l daemon.Logger = daemon.NewDummyLogger(w, logSampleRate)
 	if csvLog {
 		csvW := io.Writer(w)
 		// set up logging of CSV samples to file
@@ -107,7 +109,7 @@ func main() {
 			// write both to stderr and file
 			csvW = io.MultiWriter(w, f)
 		}
-		l = daemon.NewCSVLogger(csvW)
+		l = daemon.NewCSVLogger(csvW, logSampleRate)
 	}
 	stats := daemon.NewJSONStats()
 	go stats.Start(monitoringPort)

fbclock/daemon/logging.go

@@ -20,6 +20,7 @@ import (
 	"encoding/csv"
 	"fmt"
 	"io"
+	"math/rand"
 	"strconv"
 	"time"
 )
@@ -59,6 +60,14 @@ var header = []string{
 	"clock_accuracy_mean",
 }
 
+func shouldLog(sampleRate int) bool {
+	if sampleRate < 1 {
+		return false
+	}
+	// randomized sampling
+	return rand.Intn(sampleRate) == 0
+}
+
 // CSVRecords returns all data from this sample as CSV. Must by synced with `header` variable.
 func (s *LogSample) CSVRecords() []string {
 	return []string{
@@ -87,13 +96,15 @@ type Logger interface {
 // CSVLogger logs Sample as CSV into given writer
 type CSVLogger struct {
 	csvwriter     *csv.Writer
+	sampleRate    int
 	printedHeader bool
 }
 
 // NewCSVLogger returns new CSVLogger
-func NewCSVLogger(w io.Writer) *CSVLogger {
+func NewCSVLogger(w io.Writer, sampleRate int) *CSVLogger {
 	return &CSVLogger{
-		csvwriter: csv.NewWriter(w),
+		csvwriter:  csv.NewWriter(w),
+		sampleRate: sampleRate,
 	}
 }
 
@@ -105,6 +116,9 @@ func (l *CSVLogger) Log(s *LogSample) error {
 		}
 		l.printedHeader = true
 	}
+	if !shouldLog(l.sampleRate) {
+		return nil
+	}
 	csv := s.CSVRecords()
 	if err := l.csvwriter.Write(csv); err != nil {
 		return err
@@ -115,16 +129,20 @@ func (l *CSVLogger) Log(s *LogSample) error {
 
 // DummyLogger logs M and W to given writer
 type DummyLogger struct {
-	w io.Writer
+	w          io.Writer
+	sampleRate int
 }
 
 // NewDummyLogger returns new DummyLogger
-func NewDummyLogger(w io.Writer) *DummyLogger {
-	return &DummyLogger{w: w}
+func NewDummyLogger(w io.Writer, sampleRate int) *DummyLogger {
+	return &DummyLogger{w: w, sampleRate: sampleRate}
 }
 
 // Log implements Logger interface
 func (l *DummyLogger) Log(s *LogSample) error {
+	if !shouldLog(l.sampleRate) {
+		return nil
+	}
 	_, err := fmt.Fprintf(l.w, "m = %v, w = %v\n", time.Duration(s.MeasurementNS), time.Duration(s.WindowNS))
 	return err
 }

fbclock/daemon/logging_test.go

@@ -58,6 +58,11 @@ var testSample1 = &LogSample{
 	ClockAccuracyMean:       100.1,
 }
 
+func TestShouldLog(t *testing.T) {
+	require.False(t, shouldLog(0))
+	require.True(t, shouldLog(1))
+}
+
 func TestLogSample_CSVRecords(t *testing.T) {
 	got := testSample0.CSVRecords()
 	want := []string{"1.1", "1.2", "1.3", "1.4", "1.5", "1.6", "1.7", "1.8", "1.9", "2", "2.1", "2.2", "2.3", "25.1"}
@@ -70,7 +75,7 @@ func TestLogSample_CSVRecords(t *testing.T) {
 
 func TestCSVLogger_Log(t *testing.T) {
 	b := &bytes.Buffer{}
-	l := NewCSVLogger(b)
+	l := NewCSVLogger(b, 1)
 
 	err := l.Log(testSample0)
 	require.NoError(t, err)