Dynamic update config for logger level & metric enable (#2180)

* dynamically update logger level & metric enable * prometheus server start & shutdown * check nil * fix ci Co-authored-by: huangwenkang <642380437@qq>
apache · Jan 14, 2023 · 4c6a99b · 4c6a99b
1 parent c1a0698
commit 4c6a99b
Show file tree

Hide file tree

Showing 4 changed files with 89 additions and 26 deletions.
diff --git a/config/logger_config.go b/config/logger_config.go
@@ -170,3 +170,11 @@ func (lcb *LoggerConfigBuilder) SetZapConfig(zapConfig ZapConfig) *LoggerConfigB
 func (lcb *LoggerConfigBuilder) Build() *LoggerConfig {
 	return lcb.loggerConfig
 }
+
+// DynamicUpdateProperties dynamically update properties.
+func (lc *LoggerConfig) DynamicUpdateProperties(newLoggerConfig *LoggerConfig) {
+	if newLoggerConfig != nil && lc.ZapConfig.Level != newLoggerConfig.ZapConfig.Level {
+		lc.ZapConfig.Level = newLoggerConfig.ZapConfig.Level
+		logger.Infof("LoggerConfig's ZapConfig Level was dynamically updated, new value:%v", lc.ZapConfig.Level)
+	}
+}
diff --git a/config/metric_config.go b/config/metric_config.go
@@ -19,6 +19,7 @@ package config
 
 import (
 	"github.com/creasty/defaults"
+	"github.com/dubbogo/gost/log/logger"
 
 	"github.com/pkg/errors"
 )
@@ -81,3 +82,15 @@ func NewMetricConfigBuilder() *MetricConfigBuilder {
 func (mcb *MetricConfigBuilder) Build() *MetricConfig {
 	return mcb.metricConfig
 }
+
+// DynamicUpdateProperties dynamically update properties.
+func (mc *MetricConfig) DynamicUpdateProperties(newMetricConfig *MetricConfig) {
+	if newMetricConfig != nil {
+		if newMetricConfig.Enable != mc.Enable {
+			mc.Enable = newMetricConfig.Enable
+			logger.Infof("MetricConfig's Enable was dynamically updated, new value:%v", mc.Enable)
+
+			extension.GetMetricReporter("prometheus", mc.ToReporterConfig())
+		}
+	}
+}
diff --git a/config/root_config.go b/config/root_config.go
@@ -395,4 +395,10 @@ func (rc *RootConfig) Process(event *config_center.ConfigChangeEvent) {
 	}
 	// dynamically update consumer
 	rc.Consumer.DynamicUpdateProperties(updateRootConfig.Consumer)
+
+	// dynamically update logger
+	rc.Logger.DynamicUpdateProperties(updateRootConfig.Logger)
+
+	// dynamically update metric
+	rc.Metric.DynamicUpdateProperties(updateRootConfig.Metric)
 }
diff --git a/metrics/prometheus/reporter.go b/metrics/prometheus/reporter.go
@@ -78,6 +78,7 @@ func init() {
 // if you want to use this feature, you need to initialize your prometheus.
 // https://prometheus.io/docs/guides/go-application/
 type PrometheusReporter struct {
+	reporterServer *http.Server
 	reporterConfig *metrics.ReporterConfig
 	// report the consumer-side's rt gauge data
 	consumerRTSummaryVec *prometheus.SummaryVec
@@ -103,6 +104,10 @@ type PrometheusReporter struct {
 // the role in url must be consumer or provider
 // or it will be ignored
 func (reporter *PrometheusReporter) Report(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation, cost time.Duration, res protocol.Result) {
+	if !reporter.reporterConfig.Enable {
+		return
+	}
+
 	url := invoker.GetURL()
 	var rtVec *prometheus.SummaryVec
 	if isProvider(url) {
@@ -220,29 +225,20 @@ func newPrometheusReporter(reporterConfig *metrics.ReporterConfig) metrics.Repor
 				consumerRTSummaryVec: newSummaryVec(consumerPrefix+serviceKey+rtSuffix, reporterConfig.Namespace, labelNames, reporterConfig.SummaryMaxAge),
 				providerRTSummaryVec: newSummaryVec(providerPrefix+serviceKey+rtSuffix, reporterConfig.Namespace, labelNames, reporterConfig.SummaryMaxAge),
 			}
-			prom.DefaultRegisterer.MustRegister(reporterInstance.consumerRTSummaryVec, reporterInstance.providerRTSummaryVec)
-			metricsExporter, err := ocprom.NewExporter(ocprom.Options{
-				Registry: prom.DefaultRegisterer.(*prom.Registry),
-			})
-			if err != nil {
-				logger.Errorf("new prometheus reporter with error = %s", err)
-				return
-			}
 
-			if reporterConfig.Enable {
-				if reporterConfig.Mode == metrics.ReportModePull {
-					go func() {
-						mux := http.NewServeMux()
-						mux.Handle(reporterConfig.Path, metricsExporter)
-						if err := http.ListenAndServe(":"+reporterConfig.Port, mux); err != nil {
-							logger.Warnf("new prometheus reporter with error = %s", err)
-						}
-					}()
-				}
-				// todo pushgateway support
-			}
+			prom.DefaultRegisterer.MustRegister(reporterInstance.consumerRTSummaryVec, reporterInstance.providerRTSummaryVec)
 		})
 	}
+
+	if reporterConfig.Enable {
+		if reporterConfig.Mode == metrics.ReportModePull {
+			go reporterInstance.startupServer(reporterConfig)
+		}
+		// todo pushgateway support
+	} else {
+		reporterInstance.shutdownServer()
+	}
+
 	return reporterInstance
 }
 
@@ -377,25 +373,65 @@ func (reporter *PrometheusReporter) incSummary(summaryName string, toSetValue fl
 }
 
 func SetGaugeWithLabel(gaugeName string, val float64, label prometheus.Labels) {
-	reporterInstance.setGauge(gaugeName, val, label)
+	if reporterInstance.reporterConfig.Enable {
+		reporterInstance.setGauge(gaugeName, val, label)
+	}
 }
 
 func SetGauge(gaugeName string, val float64) {
-	reporterInstance.setGauge(gaugeName, val, make(prometheus.Labels))
+	if reporterInstance.reporterConfig.Enable {
+		reporterInstance.setGauge(gaugeName, val, make(prometheus.Labels))
+	}
 }
 
 func IncCounterWithLabel(counterName string, label prometheus.Labels) {
-	reporterInstance.incCounter(counterName, label)
+	if reporterInstance.reporterConfig.Enable {
+		reporterInstance.incCounter(counterName, label)
+	}
 }
 
 func IncCounter(summaryName string) {
-	reporterInstance.incCounter(summaryName, make(prometheus.Labels))
+	if reporterInstance.reporterConfig.Enable {
+		reporterInstance.incCounter(summaryName, make(prometheus.Labels))
+	}
 }
 
 func IncSummaryWithLabel(counterName string, val float64, label prometheus.Labels) {
-	reporterInstance.incSummary(counterName, val, label)
+	if reporterInstance.reporterConfig.Enable {
+		reporterInstance.incSummary(counterName, val, label)
+	}
 }
 
 func IncSummary(summaryName string, val float64) {
-	reporterInstance.incSummary(summaryName, val, make(prometheus.Labels))
+	if reporterInstance.reporterConfig.Enable {
+		reporterInstance.incSummary(summaryName, val, make(prometheus.Labels))
+	}
+}
+
+func (reporter *PrometheusReporter) startupServer(reporterConfig *metrics.ReporterConfig) {
+	metricsExporter, err := ocprom.NewExporter(ocprom.Options{
+		Registry: prom.DefaultRegisterer.(*prom.Registry),
+	})
+	if err != nil {
+		logger.Errorf("new prometheus reporter with error = %s", err)
+		return
+	}
+
+	// start server
+	mux := http.NewServeMux()
+	mux.Handle(reporterConfig.Path, metricsExporter)
+	reporterInstance.reporterServer = &http.Server{Addr: ":" + reporterConfig.Port, Handler: mux}
+	if err := reporterInstance.reporterServer.ListenAndServe(); err != nil {
+		logger.Warnf("new prometheus reporter with error = %s", err)
+	}
+}
+
+func (reporter *PrometheusReporter) shutdownServer() {
+	if reporterInstance.reporterServer != nil {
+		err := reporterInstance.reporterServer.Shutdown(context.Background())
+		if err != nil {
+			logger.Errorf("shutdown prometheus reporter with error = %s, prometheus reporter close now", err)
+			reporterInstance.reporterServer.Close()
+		}
+	}
 }