Merge pull request #304 from sctb512/improve-fs-metrics

metrics: improve FS metrics and remove unused metrics

pkg/metrics/collector/collector.go

@@ -30,6 +30,10 @@ func NewFsMetricsCollector(m *types.FsMetrics, imageRef string) *FsMetricsCollec
 	return &FsMetricsCollector{m, imageRef}
 }
 
+func NewFsMetricsVecCollector() *FsMetricsVecCollector {
+	return &FsMetricsVecCollector{}
+}
+
 func NewDaemonInfoCollector(version *types.BuildTimeInfo, value float64) *DaemonInfoCollector {
 	return &DaemonInfoCollector{version, value}
 }

pkg/metrics/collector/fs.go

@@ -10,20 +10,26 @@ import (
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/nydus-snapshotter/pkg/daemon/types"
 	"github.com/containerd/nydus-snapshotter/pkg/metrics/data"
+	mtypes "github.com/containerd/nydus-snapshotter/pkg/metrics/types"
 )
 
 type FsMetricsCollector struct {
 	Metrics  *types.FsMetrics
 	ImageRef string
 }
 
+type FsMetricsVecCollector struct {
+	MetricsVec []FsMetricsCollector
+}
+
 func (f *FsMetricsCollector) Collect() {
 	if f.Metrics == nil {
 		log.L.Warnf("can not collect FS metrics: Metrics is nil")
 		return
 	}
-	data.TotalRead.WithLabelValues(f.ImageRef).Set(float64(f.Metrics.DataRead))
-	data.OpenFdCount.WithLabelValues(f.ImageRef).Set(float64(f.Metrics.NrOpens))
+	data.FsTotalRead.WithLabelValues(f.ImageRef).Set(float64(f.Metrics.DataRead))
+	data.FsReadHit.WithLabelValues(f.ImageRef).Set(float64(f.Metrics.FopHits[mtypes.Read]))
+	data.FsReadError.WithLabelValues(f.ImageRef).Set(float64(f.Metrics.FopErrors[mtypes.Read]))
 
 	for _, h := range data.MetricHists {
 		o, err := h.ToConstHistogram(f.Metrics, f.ImageRef)
@@ -34,3 +40,16 @@ func (f *FsMetricsCollector) Collect() {
 		h.Save(o)
 	}
 }
+
+func (f *FsMetricsVecCollector) Clear() {
+	for _, h := range data.MetricHists {
+		h.Clear()
+	}
+}
+
+func (f *FsMetricsVecCollector) Collect() {
+	f.Clear()
+	for _, fsMetrics := range f.MetricsVec {
+		fsMetrics.Collect()
+	}
+}

pkg/metrics/data/fs.go

@@ -19,7 +19,7 @@ var (
 )
 
 var (
-	TotalRead = ttl.NewGaugeVecWithTTL(
+	FsTotalRead = ttl.NewGaugeVecWithTTL(
 		prometheus.GaugeOpts{
 			Name: "nydusd_total_read_bytes",
 			Help: "Total bytes read against the nydus filesystem",
@@ -28,10 +28,18 @@ var (
 		ttl.DefaultTTL,
 	)
 
-	OpenFdCount = ttl.NewGaugeVecWithTTL(
+	FsReadHit = ttl.NewGaugeVecWithTTL(
 		prometheus.GaugeOpts{
-			Name: "nydusd_total_open_fd_counts",
-			Help: "Total number of files that are currently open.",
+			Name: "nydusd_read_hits",
+			Help: "Total number of successful read operations.",
+		},
+		[]string{imageRefLabel},
+		ttl.DefaultTTL,
+	)
+	FsReadError = ttl.NewGaugeVecWithTTL(
+		prometheus.GaugeOpts{
+			Name: "nydusd_read_errors",
+			Help: "Total number of failed read operations.",
 		},
 		[]string{imageRefLabel},
 		ttl.DefaultTTL,
@@ -52,33 +60,6 @@ var MetricHists = []*mtypes.MetricHistogram{
 			return m.BlockCountRead
 		},
 	},
-
-	{
-		Desc: prometheus.NewDesc(
-			"nydusd_file_operation_hits",
-			"Successful various file operations histogram",
-			[]string{imageRefLabel},
-			prometheus.Labels{},
-		),
-		Buckets: mtypes.MakeFopBuckets(),
-		GetCounters: func(m *types.FsMetrics) []uint64 {
-			return m.FopHits
-		},
-	},
-
-	{
-		Desc: prometheus.NewDesc(
-			"nydusd_file_operation_errors",
-			"Failed file operations histogram",
-			[]string{imageRefLabel},
-			prometheus.Labels{},
-		),
-		Buckets: mtypes.MakeFopBuckets(),
-		GetCounters: func(m *types.FsMetrics) []uint64 {
-			return m.FopErrors
-		},
-	},
-
 	{
 		Desc: prometheus.NewDesc(
 			"nydusd_read_latency_microseconds",

pkg/metrics/registry/registry.go

@@ -17,8 +17,9 @@ var (
 
 func init() {
 	Registry.MustRegister(
-		data.TotalRead,
-		data.OpenFdCount,
+		data.FsTotalRead,
+		data.FsReadHit,
+		data.FsReadError,
 		data.NydusdEventCount,
 		data.NydusdCount,
 		data.SnapshotEventElapsedHists,

pkg/metrics/serve.go

@@ -27,7 +27,7 @@ type Server struct {
 	rootDir     string
 	metricsFile string
 	pm          *manager.Manager
-	fsCollector *collector.FsMetricsCollector
+	fsCollector *collector.FsMetricsVecCollector
 	snCollector *collector.SnapshotterMetricsCollector
 }
 
@@ -68,7 +68,7 @@ func NewServer(ctx context.Context, opts ...ServerOpt) (*Server, error) {
 		}
 	}
 
-	s.fsCollector = collector.NewFsMetricsCollector(nil, "")
+	s.fsCollector = collector.NewFsMetricsVecCollector()
 	snCollector, err := collector.NewSnapshotterMetricsCollector(ctx, s.pm.CacheDir(), os.Getpid())
 	if err != nil {
 		return nil, errors.Wrap(err, "new snapshotter metrics collector failed")
@@ -87,6 +87,7 @@ func NewServer(ctx context.Context, opts ...ServerOpt) (*Server, error) {
 func (s *Server) CollectFsMetrics(ctx context.Context) {
 	// Collect FS metrics from daemons.
 	daemons := s.pm.ListDaemons()
+	var fsMetricsVec []collector.FsMetricsCollector
 	for _, d := range daemons {
 		for _, i := range d.Instances.List() {
 			var sid string
@@ -102,13 +103,15 @@ func (s *Server) CollectFsMetrics(ctx context.Context) {
 				log.G(ctx).Errorf("failed to get fs metric: %v", err)
 				continue
 			}
-			s.fsCollector.Metrics = fsMetrics
-			s.fsCollector.ImageRef = i.ImageID
 
-			s.fsCollector.Collect()
+			fsMetricsVec = append(fsMetricsVec, collector.FsMetricsCollector{
+				Metrics:  fsMetrics,
+				ImageRef: i.ImageID,
+			})
 		}
-
 	}
+	s.fsCollector.MetricsVec = fsMetricsVec
+	s.fsCollector.Collect()
 }
 
 func (s *Server) StartCollectMetrics(ctx context.Context) error {

pkg/metrics/types/types.go

@@ -56,7 +56,7 @@ type MetricHistogram struct {
 	GetCounters GetCountersFn
 
 	// Save the last generated histogram metric
-	constHist prometheus.Metric
+	constHists []prometheus.Metric
 }
 
 func (h *MetricHistogram) ToConstHistogram(m *types.FsMetrics, imageRef string) (prometheus.Metric, error) {
@@ -71,7 +71,7 @@ func (h *MetricHistogram) ToConstHistogram(m *types.FsMetrics, imageRef string)
 	for i, c := range counters {
 		count += c
 		sum += h.Buckets[i] * c
-		hmap[float64(h.Buckets[i])] = c
+		hmap[float64(h.Buckets[i])] = count
 	}
 
 	return prometheus.MustNewConstHistogram(
@@ -82,8 +82,12 @@ func (h *MetricHistogram) ToConstHistogram(m *types.FsMetrics, imageRef string)
 	), nil
 }
 
+func (h *MetricHistogram) Clear() {
+	h.constHists = nil
+}
+
 func (h *MetricHistogram) Save(m prometheus.Metric) {
-	h.constHist = m
+	h.constHists = append(h.constHists, m)
 }
 
 // Implement prometheus.Collector interface
@@ -94,7 +98,9 @@ func (h *MetricHistogram) Describe(ch chan<- *prometheus.Desc) {
 }
 
 func (h *MetricHistogram) Collect(ch chan<- prometheus.Metric) {
-	if h.constHist != nil {
-		ch <- h.constHist
+	if h.constHists != nil {
+		for _, hist := range h.constHists {
+			ch <- hist
+		}
 	}
 }