Add support for unicast keepalived

With this PR keepalived capability extended to support both unicast and multicast modes.

The following components were updated to support unicast mode:

- A simple server that will return node's non-virtual IP address, this server will be used
  by the master nodes to retrieve the bootstrap node IP address. One should contact the server on port 64444.

- Update keepalived-moniror to render relevant nodes IP addresses in keepalived.conf file to enable unicast VRRP packets.
  An important point to emphasize, for master nodes we shouldn't use information read from kube-apiserver through api-vip for
  keepalived conf rendering (egg and chicken problem), instead of that we should communicate with localhost:kube-apiserver

.gitignore

@@ -48,3 +48,4 @@ network_closure.sh
 /monitor
 /dynkeepalived
 /corednsmonitor
+/unicastipserver

Dockerfile

@@ -5,12 +5,14 @@ RUN GO111MODULE=on go build --mod=vendor -o runtimecfg ./cmd/runtimecfg
 RUN GO111MODULE=on go build --mod=vendor cmd/dynkeepalived/dynkeepalived.go
 RUN GO111MODULE=on go build --mod=vendor cmd/corednsmonitor/corednsmonitor.go
 RUN GO111MODULE=on go build --mod=vendor cmd/monitor/monitor.go
+RUN GO111MODULE=on go build --mod=vendor cmd/unicastipserver/unicastipserver.go
 
 FROM registry.svc.ci.openshift.org/openshift/origin-v4.0:base
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/runtimecfg /usr/bin/
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/monitor /usr/bin
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/dynkeepalived /usr/bin
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/corednsmonitor /usr/bin
+COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/unicastipserver /usr/bin
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/scripts/* /usr/bin/
 
 ENTRYPOINT ["/usr/bin/runtimecfg"]

cmd/unicastipserver/unicastipserver.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/openshift/baremetal-runtimecfg/pkg/monitor"
+	"github.com/spf13/cobra"
+)
+
+func main() {
+	var rootCmd = &cobra.Command{
+		Use:   "unicastipserver [path to kubeconfig] [api-vip] [dns-vip] [ingress-vip] [api-provisioning-vip]",
+		Short: "baremetal-runtimecfg discovers OpenShift cluster and node configuration and renders Go templates",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			apiVip, err := cmd.Flags().GetIP("api-vip")
+			if err != nil {
+				apiVip = nil
+			}
+			ingressVip, err := cmd.Flags().GetIP("ingress-vip")
+			if err != nil {
+				ingressVip = nil
+			}
+			dnsVip, err := cmd.Flags().GetIP("dns-vip")
+			if err != nil {
+				dnsVip = nil
+			}
+			unicastipServerPort, err := cmd.Flags().GetUint16("unicastip-server-port")
+			if err != nil {
+				return err
+			}
+
+			return monitor.UnicastIPServer(apiVip, ingressVip, dnsVip, unicastipServerPort)
+		},
+	}
+
+	rootCmd.Flags().IP("api-vip", nil, "Virtual IP Address to reach the OpenShift API")
+	rootCmd.Flags().IP("ingress-vip", nil, "Virtual IP Address to reach the OpenShift Ingress Routers")
+	rootCmd.Flags().IP("dns-vip", nil, "Virtual IP Address to reach an OpenShift node resolving DNS server")
+	rootCmd.Flags().Uint16("unicastip-server-port", 64444, "Port where the OpenShift API listens at")
+
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

go.sum

@@ -37,6 +37,7 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=

pkg/config/node.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/sirupsen/logrus"
@@ -22,6 +23,9 @@ import (
 	"github.com/openshift/installer/pkg/types"
 )
 
+const bootstrapIpServerPort string = "64444"
+const localhostKubeApiServerUrl string = "https://localhost:6443"
+
 var log = logrus.New()
 
 type Cluster struct {
@@ -51,6 +55,10 @@ type ApiLBConfig struct {
 	FrontendAddr string
 }
 
+type IngressConfig struct {
+	Peers []string
+}
+
 type Node struct {
 	Cluster           Cluster
 	LBConfig          ApiLBConfig
@@ -59,6 +67,9 @@ type Node struct {
 	EtcdShortHostname string
 	VRRPInterface     string
 	DNSUpstreams      []string
+	BootstrapIP       string
+	IngressConfig     IngressConfig
+	//EnableUnicast     bool
 }
 
 func getDNSUpstreams(resolvConfPath string) (upstreams []string, err error) {
@@ -166,6 +177,67 @@ func (c *Cluster) PopulateVRIDs() error {
 	}
 	return nil
 }
+func GetBootstrapIP(apiVip string) (bootstrapIP string, err error) {
+	conn, err := net.DialTimeout("tcp", net.JoinHostPort(apiVip, bootstrapIpServerPort), 10*time.Second)
+	if err != nil {
+		log.Infof("An error occurred on dial: %v", err)
+		return "", err
+	}
+	defer conn.Close()
+
+	bootstrapIP, err = bufio.NewReader(conn).ReadString('\n')
+	if err != nil {
+		log.Infof("An error occurred on read: %v", err)
+		return "", err
+	}
+
+	bootstrapIP = strings.TrimSpace(bootstrapIP)
+
+	log.Infof("Got bootstrap IP %v", bootstrapIP)
+
+	return bootstrapIP, err
+}
+
+func GetVRRPConfig(apiVip, ingressVip, dnsVip net.IP) (vipIface net.Interface, nonVipAddr *net.IPNet, err error) {
+	vips := make([]net.IP, 0)
+	if apiVip != nil {
+		vips = append(vips, apiVip)
+	}
+	if ingressVip != nil {
+		vips = append(vips, ingressVip)
+	}
+	if dnsVip != nil {
+		vips = append(vips, dnsVip)
+	}
+	return getInterfaceAndNonVIPAddr(vips)
+}
+
+func GetIngressConfig(kubeconfigPath string) (ingressConfig IngressConfig, err error) {
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	if err != nil {
+		return ingressConfig, err
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return ingressConfig, err
+	}
+
+	nodes, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return ingressConfig, err
+	}
+
+	for _, node := range nodes.Items {
+		for _, address := range node.Status.Addresses {
+			if address.Type == v1.NodeInternalIP {
+				ingressConfig.Peers = append(ingressConfig.Peers, address.Address)
+			}
+		}
+	}
+
+	return ingressConfig, nil
+}
 
 func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, dnsVip net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) {
 	// Try cluster-config.yml first
@@ -202,25 +274,26 @@ func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip
 		return node, err
 	}
 
-	vips := make([]net.IP, 0)
 	if apiVip != nil {
-		vips = append(vips, apiVip)
 		node.Cluster.APIVIP = apiVip.String()
 	}
 	if ingressVip != nil {
-		vips = append(vips, ingressVip)
 		node.Cluster.IngressVIP = ingressVip.String()
 	}
 	if dnsVip != nil {
-		vips = append(vips, dnsVip)
 		node.Cluster.DNSVIP = dnsVip.String()
 	}
-	vipIface, nonVipAddr, err := getInterfaceAndNonVIPAddr(vips)
+	vipIface, nonVipAddr, err := GetVRRPConfig(apiVip, ingressVip, dnsVip)
 	if err != nil {
 		return node, err
 	}
 	node.NonVirtualIP = nonVipAddr.IP.String()
-
+	/*
+		node.EnableUnicast = false
+		if os.Getenv("ENABLE_UNICAST") == "yes" {
+			node.EnableUnicast = true
+		}
+	*/
 	resolvConfUpstreams, err := getDNSUpstreams(resolvConfPath)
 	if err != nil {
 		return node, err
@@ -245,9 +318,9 @@ func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip
 	return node, err
 }
 
-func getSortedBackends(kubeconfigPath string) (backends []Backend, err error) {
+func getSortedBackends(kubeconfigPath string, kubeApiServerUrl string) (backends []Backend, err error) {
 
-	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	config, err := clientcmd.BuildConfigFromFlags(kubeApiServerUrl, kubeconfigPath)
 	if err != nil {
 		log.WithFields(logrus.Fields{
 			"err": err,
@@ -291,18 +364,31 @@ func getSortedBackends(kubeconfigPath string) (backends []Backend, err error) {
 	return backends, err
 }
 
-func GetLBConfig(kubeconfigPath string, apiPort, lbPort, statPort uint16, apiVip net.IP) (ApiLBConfig, error) {
+func GetLBConfig(kubeconfigPath string, apiPort, lbPort, statPort uint16, apiVip net.IP, queyrAlsoApiVipOpt ...bool) (ApiLBConfig, error) {
 	config := ApiLBConfig{
 		ApiPort:  apiPort,
 		LbPort:   lbPort,
 		StatPort: statPort,
 	}
+	queyrAlsoApiVip := true
+	if len(queyrAlsoApiVipOpt) > 0 {
+		queyrAlsoApiVip = queyrAlsoApiVipOpt[0]
+	}
+
 	// LB frontend address: IPv6 '::' , IPv4 ''
 	if apiVip.To4() == nil {
 		config.FrontendAddr = "::"
 	}
-
-	backends, err := getSortedBackends(kubeconfigPath)
+	// Since master nodes details used for rendering api-vip keeplived config (in uncast mode) we shouldn't read it through api-vip
+	// (egg and chicken problem), so we'll first try read it from localhost:kube-apiserver and only failover to api-vip:kube-apiserver
+	backends, err := getSortedBackends(kubeconfigPath, localhostKubeApiServerUrl)
+	if err != nil {
+		log.Infof("An error occurred while trying to read master nodes details from localhost:kube-apiserver: %v", err)
+		if queyrAlsoApiVip {
+			log.Infof("Trying to read master nodes details from api-vip:kube-apiserver")
+			backends, err = getSortedBackends(kubeconfigPath, "")
+		}
+	}
 	if err != nil {
 		log.WithFields(logrus.Fields{
 			"kubeconfigPath": kubeconfigPath,