wal: add "etcd_wal_writes_bytes_total"

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>

wal/encoder.go

@@ -92,7 +92,8 @@ func (e *encoder) encode(rec *walpb.Record) error {
 	if padBytes != 0 {
 		data = append(data, make([]byte, padBytes)...)
 	}
-	_, err = e.bw.Write(data)
+	n, err = e.bw.Write(data)
+	walWriteBytes.Add(float64(n))
 	return err
 }
 
@@ -108,13 +109,16 @@ func encodeFrameSize(dataBytes int) (lenField uint64, padBytes int) {
 
 func (e *encoder) flush() error {
 	e.mu.Lock()
-	defer e.mu.Unlock()
-	return e.bw.Flush()
+	n, err := e.bw.FlushN()
+	e.mu.Unlock()
+	walWriteBytes.Add(float64(n))
+	return err
 }
 
 func writeUint64(w io.Writer, n uint64, buf []byte) error {
 	// http://golang.org/src/encoding/binary/binary.go
 	binary.LittleEndian.PutUint64(buf, n)
-	_, err := w.Write(buf)
+	nv, err := w.Write(buf)
+	walWriteBytes.Add(float64(nv))
 	return err
 }

wal/metrics.go

@@ -24,8 +24,15 @@ var (
 		Help:      "The latency distributions of fsync called by wal.",
 		Buckets:   prometheus.ExponentialBuckets(0.001, 2, 14),
 	})
+	walWriteBytes = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "etcd",
+		Subsystem: "disk",
+		Name:      "wal_write_bytes_total",
+		Help:      "Total number of bytes written in WAL.",
+	})
 )
 
 func init() {
 	prometheus.MustRegister(syncDurations)
+	prometheus.MustRegister(walWriteBytes)
 }

wal/repair.go

@@ -18,6 +18,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/coreos/etcd/pkg/fileutil"
 	"github.com/coreos/etcd/wal/walpb"
@@ -76,10 +77,14 @@ func Repair(dirpath string) bool {
 				plog.Errorf("could not repair %v, failed to truncate file", f.Name())
 				return false
 			}
+
+			start := time.Now()
 			if err = fileutil.Fsync(f.File); err != nil {
 				plog.Errorf("could not repair %v, failed to sync file", f.Name())
 				return false
 			}
+			syncDurations.Observe(time.Since(start).Seconds())
+
 			return true
 		default:
 			plog.Errorf("could not repair error (%v)", err)

wal/wal.go

@@ -147,9 +147,13 @@ func Create(dirpath string, metadata []byte) (*WAL, error) {
 	if perr != nil {
 		return nil, perr
 	}
+
+	start := time.Now()
 	if perr = fileutil.Fsync(pdir); perr != nil {
 		return nil, perr
 	}
+	syncDurations.Observe(time.Since(start).Seconds())
+
 	if perr = pdir.Close(); err != nil {
 		return nil, perr
 	}
@@ -548,9 +552,12 @@ func (w *WAL) cut() error {
 	if err = os.Rename(newTail.Name(), fpath); err != nil {
 		return err
 	}
+
+	start := time.Now()
 	if err = fileutil.Fsync(w.dirFile); err != nil {
 		return err
 	}
+	syncDurations.Observe(time.Since(start).Seconds())
 
 	// reopen newTail with its new path so calls to Name() match the wal filename format
 	newTail.Close()