Merge cad4219 into 444007f

exporter.go

@@ -21,6 +21,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"sync"
 	"syscall"
 	"time"
@@ -134,10 +135,12 @@ func (c *CephExporter) Collect(ch chan<- prometheus.Metric) {
 
 func main() {
 	var (
-		addr        = flag.String("telemetry.addr", ":9128", "host:port for ceph exporter")
-		metricsPath = flag.String("telemetry.path", "/metrics", "URL path for surfacing collected metrics")
-		cephConfig  = flag.String("ceph.config", "", "path to ceph config file")
-		cephUser    = flag.String("ceph.user", "admin", "Ceph user to connect to cluster.")
+		addr                  = flag.String("telemetry.addr", ":9128", "host:port for ceph exporter")
+		metricsPath           = flag.String("telemetry.path", "/metrics", "URL path for surfacing collected metrics")
+		cephConfig            = flag.String("ceph.config", "", "path to Ceph config file")
+		cephUser              = flag.String("ceph.user", "admin", "Ceph user to connect to cluster.")
+		cephRadosOSDOpTimeout = flag.Int64("ceph.rados_osd_op_timeout_sec", 30, "Ceph rados_osd_op_timeout used to contact cluster in seconds. 0 means on limit")
+		cephRadosMonOpTimeout = flag.Int64("ceph.rados_mon_op_timeout_sec", 30, "Ceph rados_mon_op_timeout used to contact cluster in seconds. 0 means on limit")
 
 		rgwMode = flag.Int("rgw.mode", 0, "Enable collection of stats from RGW (0:disabled 1:enabled 2:background)")
 
@@ -170,6 +173,17 @@ func main() {
 			// defer Shutdown to program exit
 			defer conn.Shutdown()
 
+			// Set rados_osd_op_timeout and rados_mon_op_timeout to avoid Mon
+			// and PG command hang.
+			// See https://github.com/ceph/ceph/blob/d4872ce97a2825afcb58876559cc73aaa1862c0f/src/common/legacy_config_opts.h#L1258-L1259
+			if err := conn.SetConfigOption("rados_osd_op_timeout", strconv.FormatInt(*cephRadosOSDOpTimeout, 10)); err != nil {
+				log.Fatalf("cannot set rados_osd_op_timeout for ceph cluster: %s", err)
+			}
+
+			if err := conn.SetConfigOption("rados_mon_op_timeout", strconv.FormatInt(*cephRadosMonOpTimeout, 10)); err != nil {
+				log.Fatalf("cannot set rados_mon_op_timeout for ceph cluster: %s", err)
+			}
+
 			log.Printf("Starting ceph exporter for cluster: %s", cluster.ClusterLabel)
 			err = prometheus.Register(NewCephExporter(conn, cluster.ClusterLabel, cluster.ConfigFile, *rgwMode))
 			if err != nil {
@@ -196,6 +210,17 @@ func main() {
 		}
 		defer conn.Shutdown()
 
+		// Set rados_osd_op_timeout and rados_mon_op_timeout to avoid Mon
+		// and PG command hang.
+		// See https://github.com/ceph/ceph/blob/d4872ce97a2825afcb58876559cc73aaa1862c0f/src/common/legacy_config_opts.h#L1258-L1259
+		if err := conn.SetConfigOption("rados_osd_op_timeout", strconv.FormatInt(*cephRadosOSDOpTimeout, 10)); err != nil {
+			log.Fatalf("cannot set rados_osd_op_timeout for ceph cluster: %s", err)
+		}
+
+		if err := conn.SetConfigOption("rados_mon_op_timeout", strconv.FormatInt(*cephRadosMonOpTimeout, 10)); err != nil {
+			log.Fatalf("cannot set rados_mon_op_timeout for ceph cluster: %s", err)
+		}
+
 		prometheus.MustRegister(NewCephExporter(conn, defaultCephClusterLabel, defaultCephConfigPath, *rgwMode))
 	}