Fix monitoring metrics (#25549) (#25659)

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
milvus-io · Jul 18, 2023 · 0567128 · 0567128
1 parent f939bd0
commit 0567128
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 9 deletions.
diff --git a/internal/rootcoord/meta_table.go b/internal/rootcoord/meta_table.go
@@ -135,6 +135,9 @@ func (mt *MetaTable) reload() error {
 	collectionNum := int64(0)
 	partitionNum := int64(0)
 
+	metrics.RootCoordNumOfCollections.Set(float64(0))
+	metrics.RootCoordNumOfPartitions.WithLabelValues().Set(float64(0))
+
 	// recover databases.
 	dbs, err := mt.catalog.ListDatabases(mt.ctx, typeutil.MaxTimestamp)
 	if err != nil {
@@ -193,8 +196,8 @@ func (mt *MetaTable) reload() error {
 		}
 	}
 
-	metrics.RootCoordNumOfCollections.Set(float64(collectionNum))
-	metrics.RootCoordNumOfPartitions.WithLabelValues().Set(float64(partitionNum))
+	metrics.RootCoordNumOfCollections.Add(float64(collectionNum))
+	metrics.RootCoordNumOfPartitions.WithLabelValues().Add(float64(partitionNum))
 
 	log.Info("meta table recovery finished")
 	return nil
@@ -229,7 +232,10 @@ func (mt *MetaTable) reloadWithNonDatabase() error {
 	for _, alias := range aliases {
 		mt.aliases.insert(util.DefaultDBName, alias.Name, alias.CollectionID)
 	}
-	return err
+
+	metrics.RootCoordNumOfCollections.Add(float64(collectionNum))
+	metrics.RootCoordNumOfPartitions.WithLabelValues().Add(float64(partitionNum))
+	return nil
 }
 
 func (mt *MetaTable) createDefaultDb() error {

diff --git a/internal/rootcoord/proxy_client_manager.go b/internal/rootcoord/proxy_client_manager.go
@@ -28,6 +28,7 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
 	"github.com/milvus-io/milvus/internal/log"
+	"github.com/milvus-io/milvus/internal/metrics"
 	"github.com/milvus-io/milvus/internal/proto/proxypb"
 	"github.com/milvus-io/milvus/internal/types"
 	"github.com/milvus-io/milvus/internal/util/metricsinfo"
@@ -74,6 +75,7 @@ func (p *proxyClientManager) AddProxyClient(session *sessionutil.Session) {
 	}
 
 	p.connect(session)
+	p.updateProxyNumMetric()
 }
 
 // GetProxyNumber returns number of proxy clients.
@@ -84,6 +86,11 @@ func (p *proxyClientManager) GetProxyNumber() int {
 	return len(p.proxyClient)
 }
 
+// mutex.Lock is required before calling this method.
+func (p *proxyClientManager) updateProxyNumMetric() {
+	metrics.RootCoordProxyCounter.WithLabelValues().Set(float64(len(p.proxyClient)))
+}
+
 func (p *proxyClientManager) connect(session *sessionutil.Session) {
 	pc, err := p.creator(session)
 	if err != nil {
@@ -114,6 +121,7 @@ func (p *proxyClientManager) DelProxyClient(s *sessionutil.Session) {
 	}
 
 	delete(p.proxyClient, s.ServerID)
+	p.updateProxyNumMetric()
 	log.Info("remove proxy client", zap.String("proxy address", s.Address), zap.Int64("proxy id", s.ServerID))
 }
 

diff --git a/internal/rootcoord/proxy_manager.go b/internal/rootcoord/proxy_manager.go
@@ -30,7 +30,6 @@ import (
 
 	etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
 	"github.com/milvus-io/milvus/internal/log"
-	"github.com/milvus-io/milvus/internal/metrics"
 	"github.com/milvus-io/milvus/internal/util/sessionutil"
 	"github.com/milvus-io/milvus/internal/util/typeutil"
 )
@@ -153,7 +152,6 @@ func (p *proxyManager) handlePutEvent(e *clientv3.Event) error {
 	for _, f := range p.addSessionsFunc {
 		f(session)
 	}
-	metrics.RootCoordProxyCounter.WithLabelValues().Inc()
 	return nil
 }
 
@@ -166,7 +164,6 @@ func (p *proxyManager) handleDeleteEvent(e *clientv3.Event) error {
 	for _, f := range p.delSessionsFunc {
 		f(session)
 	}
-	metrics.RootCoordProxyCounter.WithLabelValues().Dec()
 	return nil
 }
 

diff --git a/internal/rootcoord/quota_center.go b/internal/rootcoord/quota_center.go
@@ -803,15 +803,15 @@ func (q *QuotaCenter) setRates() error {
 // recordMetrics records metrics of quota states.
 func (q *QuotaCenter) recordMetrics() {
 	record := func(errorCode commonpb.ErrorCode) {
+		var hasException float64
 		for _, states := range q.quotaStates {
 			for _, state := range states {
 				if state == errorCode {
-					metrics.RootCoordQuotaStates.WithLabelValues(errorCode.String()).Set(1)
+					hasException = 1
 				}
 			}
-			return
 		}
-		metrics.RootCoordQuotaStates.WithLabelValues(errorCode.String()).Set(0)
+		metrics.RootCoordQuotaStates.WithLabelValues(errorCode.String()).Set(hasException)
 	}
 	record(commonpb.ErrorCode_MemoryQuotaExhausted)
 	record(commonpb.ErrorCode_DiskQuotaExhausted)