fix: nbctl need timeout to avoid hang infinitely

(cherry picked from commit d3c6a71)
kubeovn · Nov 29, 2019 · e0fbfea · e0fbfea
1 parent dd63c5a
commit e0fbfea
Show file tree

Hide file tree

Showing 8 changed files with 32 additions and 17 deletions.
diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"net/http"
 	_ "net/http/pprof"
 	"time"
@@ -34,8 +35,12 @@ func main() {
 	}
 
 	stopCh := signals.SetupSignalHandler()
-	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(config.KubeClient, time.Second*30)
-	kubeovnInformerFactory := kubeovninformer.NewSharedInformerFactoryWithOptions(config.KubeOvnClient, time.Second*30)
+	kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(config.KubeClient, 0,
+		kubeinformers.WithTweakListOptions(func(listOption *v1.ListOptions) {
+			listOption.FieldSelector = fmt.Sprintf("spec.nodeName=%s", config.NodeName)
+			listOption.AllowWatchBookmarks = true
+		}))
+	kubeovnInformerFactory := kubeovninformer.NewSharedInformerFactoryWithOptions(config.KubeOvnClient, 0)
 	ctl, err := daemon.NewController(config, kubeInformerFactory, kubeovnInformerFactory)
 	if err != nil {
 		klog.Fatalf("create controller failed %v", err)

diff --git a/cmd/webhook/server.go b/cmd/webhook/server.go
@@ -34,13 +34,15 @@ func init() {
 
 func main() {
 	var (
-		port      int
-		ovnNbHost string
-		ovnNbPort int
-		defaultLS string
+		port         int
+		ovnNbHost    string
+		ovnNbPort    int
+		ovnNbTimeout int
+		defaultLS    string
 	)
 	flag.IntVar(&port, "port", 8443, "The port webhook listen on.")
 	flag.IntVar(&ovnNbPort, "ovn-nb-port", 6641, "OVN nb port")
+	flag.IntVar(&ovnNbTimeout, "ovn-nb-timeout", 30, "OVN nb timeout")
 	flag.StringVar(&ovnNbHost, "ovn-nb-host", "0.0.0.0", "OVN nb host")
 	flag.StringVar(&defaultLS, "default-ls", "ovn-default", "The default logical switch name, default: ovn-default")
 
@@ -66,9 +68,10 @@ func main() {
 	}
 
 	opt := &ovnwebhook.WebhookOptions{
-		OvnNbHost: ovnNbHost,
-		OvnNbPort: ovnNbPort,
-		DefaultLS: defaultLS,
+		OvnNbHost:    ovnNbHost,
+		OvnNbPort:    ovnNbPort,
+		OvnNbTimeout: ovnNbTimeout,
+		DefaultLS:    defaultLS,
 	}
 	validatingHook, err := ovnwebhook.NewValidatingHook(mgr.GetCache(), opt)
 	if err != nil {

diff --git a/pkg/controller/config.go b/pkg/controller/config.go
@@ -19,6 +19,7 @@ type Configuration struct {
 	OvnNbSocket    string
 	OvnNbHost      string
 	OvnNbPort      int
+	OvnNbTimeout   int
 	KubeConfigFile string
 	KubeClient     kubernetes.Interface
 	KubeOvnClient  clientset.Interface
@@ -50,6 +51,7 @@ func ParseFlags() (*Configuration, error) {
 		argOvnNbSocket    = pflag.String("ovn-nb-socket", "", "The ovn-nb socket file. (If not set use ovn-nb-address)")
 		argOvnNbHost      = pflag.String("ovn-nb-host", "0.0.0.0", "The ovn-nb host address. (If not set use ovn-nb-socket)")
 		argOvnNbPort      = pflag.Int("ovn-nb-port", 6641, "")
+		argOvnNbTimeout   = pflag.Int("ovn-nb-timeout", 30, "")
 		argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.")
 
 		argDefaultLogicalSwitch = pflag.String("default-ls", "ovn-default", "The default logical switch name, default: ovn-default")
@@ -91,6 +93,7 @@ func ParseFlags() (*Configuration, error) {
 		OvnNbSocket:            *argOvnNbSocket,
 		OvnNbHost:              *argOvnNbHost,
 		OvnNbPort:              *argOvnNbPort,
+		OvnNbTimeout:           *argOvnNbTimeout,
 		KubeConfigFile:         *argKubeConfigFile,
 		DefaultLogicalSwitch:   *argDefaultLogicalSwitch,
 		DefaultCIDR:            *argDefaultCIDR,

diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
@@ -117,7 +117,7 @@ func NewController(config *Configuration) *Controller {
 
 	controller := &Controller{
 		config:    config,
-		ovnClient: ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR),
+		ovnClient: ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.OvnNbTimeout, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR),
 
 		subnetsLister:           subnetInformer.Lister(),
 		subnetSynced:            subnetInformer.Informer().HasSynced,

diff --git a/pkg/controller/init.go b/pkg/controller/init.go
@@ -68,7 +68,7 @@ func InitNodeSwitch(config *Configuration) error {
 
 // InitClusterRouter init cluster router to connect different logical switches
 func InitClusterRouter(config *Configuration) error {
-	client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR)
+	client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.OvnNbTimeout, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR)
 	lrs, err := client.ListLogicalRouter()
 	if err != nil {
 		return err
@@ -84,7 +84,7 @@ func InitClusterRouter(config *Configuration) error {
 
 // InitLoadBalancer init the default tcp and udp cluster loadbalancer
 func InitLoadBalancer(config *Configuration) error {
-	client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR)
+	client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.OvnNbTimeout, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR)
 	tcpLb, err := client.FindLoadbalancer(config.ClusterTcpLoadBalancer)
 	if err != nil {
 		return fmt.Errorf("failed to find tcp lb %v", err)

diff --git a/pkg/ovs/ovn-nbctl.go b/pkg/ovs/ovn-nbctl.go
@@ -16,6 +16,7 @@ import (
 
 func (c Client) ovnNbCommand(cmdArgs ...string) (string, error) {
 	start := time.Now()
+	cmdArgs = append([]string{fmt.Sprintf("--timeout=%d", c.OvnNbTimeout)}, cmdArgs...)
 	raw, err := exec.Command(OvnNbCtl, cmdArgs...).CombinedOutput()
 	elapsed := float64((time.Since(start)) / time.Millisecond)
 	klog.Infof("%s command %s in %vms", OvnNbCtl, strings.Join(cmdArgs, " "), elapsed)

diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go
@@ -13,6 +13,7 @@ var (
 // Client is the ovn client
 type Client struct {
 	OvnNbAddress           string
+	OvnNbTimeout           int
 	OvnSbAddress           string
 	ClusterRouter          string
 	ClusterTcpLoadBalancer string
@@ -31,10 +32,11 @@ const (
 )
 
 // NewClient init an ovn client
-func NewClient(ovnNbHost string, ovnNbPort int, ovnSbHost string, ovnSbPort int, clusterRouter, clusterTcpLoadBalancer, clusterUdpLoadBalancer, nodeSwitch, nodeSwitchCIDR string) *Client {
+func NewClient(ovnNbHost string, ovnNbPort int, ovnNbTimeout int, ovnSbHost string, ovnSbPort int, clusterRouter, clusterTcpLoadBalancer, clusterUdpLoadBalancer, nodeSwitch, nodeSwitchCIDR string) *Client {
 	return &Client{
 		OvnNbAddress:           fmt.Sprintf("tcp:%s:%d", ovnNbHost, ovnNbPort),
 		OvnSbAddress:           fmt.Sprintf("tcp:%s:%d", ovnSbHost, ovnSbPort),
+		OvnNbTimeout:           ovnNbTimeout,
 		ClusterRouter:          clusterRouter,
 		ClusterTcpLoadBalancer: clusterTcpLoadBalancer,
 		ClusterUdpLoadBalancer: clusterUdpLoadBalancer,

diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go
@@ -29,9 +29,10 @@ type ValidatingHook struct {
 }
 
 type WebhookOptions struct {
-	OvnNbHost string
-	OvnNbPort int
-	DefaultLS string
+	OvnNbHost    string
+	OvnNbPort    int
+	OvnNbTimeout int
+	DefaultLS    string
 }
 
 func NewValidatingHook(c cache.Cache, opt *WebhookOptions) (*ValidatingHook, error) {
@@ -49,7 +50,7 @@ func NewValidatingHook(c cache.Cache, opt *WebhookOptions) (*ValidatingHook, err
 
 	return &ValidatingHook{
 		kubeclientset: kubeClient,
-		ovnClient:     ovs.NewClient(opt.OvnNbHost, opt.OvnNbPort, "", 0, "", "", "", "", ""),
+		ovnClient:     ovs.NewClient(opt.OvnNbHost, opt.OvnNbPort, opt.OvnNbTimeout, "", 0, "", "", "", "", ""),
 		opt:           opt,
 		cache:         c,
 	}, nil