Merge pull request #65 from yboaron/unicast_latest

Add support for unicast keepalived

.gitignore

@@ -48,3 +48,4 @@ network_closure.sh
 /monitor
 /dynkeepalived
 /corednsmonitor
+/unicastipserver

Dockerfile

@@ -5,12 +5,14 @@ RUN GO111MODULE=on go build --mod=vendor -o runtimecfg ./cmd/runtimecfg
 RUN GO111MODULE=on go build --mod=vendor cmd/dynkeepalived/dynkeepalived.go
 RUN GO111MODULE=on go build --mod=vendor cmd/corednsmonitor/corednsmonitor.go
 RUN GO111MODULE=on go build --mod=vendor cmd/monitor/monitor.go
+RUN GO111MODULE=on go build --mod=vendor cmd/unicastipserver/unicastipserver.go
 
 FROM registry.svc.ci.openshift.org/openshift/origin-v4.0:base
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/runtimecfg /usr/bin/
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/monitor /usr/bin
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/dynkeepalived /usr/bin
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/corednsmonitor /usr/bin
+COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/unicastipserver /usr/bin
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/scripts/* /usr/bin/
 COPY --from=builder /go/src/github.com/openshift/baremetal-runtimecfg/scripts/ip*tables /usr/sbin/
 

cmd/unicastipserver/unicastipserver.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/openshift/baremetal-runtimecfg/pkg/monitor"
+	"github.com/spf13/cobra"
+)
+
+func main() {
+	var rootCmd = &cobra.Command{
+		Use:   "unicastipserver [path to kubeconfig] [api-vip] [dns-vip] [ingress-vip] [api-provisioning-vip]",
+		Short: "baremetal-runtimecfg discovers OpenShift cluster and node configuration and renders Go templates",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			apiVip, err := cmd.Flags().GetIP("api-vip")
+			if err != nil {
+				apiVip = nil
+			}
+			ingressVip, err := cmd.Flags().GetIP("ingress-vip")
+			if err != nil {
+				ingressVip = nil
+			}
+			dnsVip, err := cmd.Flags().GetIP("dns-vip")
+			if err != nil {
+				dnsVip = nil
+			}
+			unicastipServerPort, err := cmd.Flags().GetUint16("unicastip-server-port")
+			if err != nil {
+				return err
+			}
+
+			return monitor.UnicastIPServer(apiVip, ingressVip, dnsVip, unicastipServerPort)
+		},
+	}
+
+	rootCmd.Flags().IP("api-vip", nil, "Virtual IP Address to reach the OpenShift API")
+	rootCmd.Flags().IP("ingress-vip", nil, "Virtual IP Address to reach the OpenShift Ingress Routers")
+	rootCmd.Flags().IP("dns-vip", nil, "Virtual IP Address to reach an OpenShift node resolving DNS server")
+	rootCmd.Flags().Uint16("unicastip-server-port", 64444, "Port where the OpenShift API listens at")
+
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

go.sum

@@ -37,6 +37,7 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=

pkg/config/node.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/sirupsen/logrus"
@@ -22,6 +23,9 @@ import (
 	"github.com/openshift/installer/pkg/types"
 )
 
+const localhostKubeApiServerUrl string = "https://localhost:6443"
+const bootstrapIpServerPort string = "64444"
+
 var log = logrus.New()
 
 type Cluster struct {
@@ -55,6 +59,10 @@ type ApiLBConfig struct {
 	FrontendAddr string
 }
 
+type IngressConfig struct {
+	Peers []string
+}
+
 type Node struct {
 	Cluster           Cluster
 	LBConfig          ApiLBConfig
@@ -63,6 +71,9 @@ type Node struct {
 	EtcdShortHostname string
 	VRRPInterface     string
 	DNSUpstreams      []string
+	BootstrapIP       string
+	IngressConfig     IngressConfig
+	EnableUnicast     bool
 }
 
 func getDNSUpstreams(resolvConfPath string) (upstreams []string, err error) {
@@ -170,6 +181,67 @@ func (c *Cluster) PopulateVRIDs() error {
 	}
 	return nil
 }
+func GetBootstrapIP(apiVip string) (bootstrapIP string, err error) {
+	conn, err := net.DialTimeout("tcp", net.JoinHostPort(apiVip, bootstrapIpServerPort), 10*time.Second)
+	if err != nil {
+		log.Infof("An error occurred on dial: %v", err)
+		return "", err
+	}
+	defer conn.Close()
+
+	bootstrapIP, err = bufio.NewReader(conn).ReadString('\n')
+	if err != nil {
+		log.Infof("An error occurred on read: %v", err)
+		return "", err
+	}
+
+	bootstrapIP = strings.TrimSpace(bootstrapIP)
+
+	log.Infof("Got bootstrap IP %v", bootstrapIP)
+
+	return bootstrapIP, err
+}
+
+func GetVRRPConfig(apiVip, ingressVip, dnsVip net.IP) (vipIface net.Interface, nonVipAddr *net.IPNet, err error) {
+	vips := make([]net.IP, 0)
+	if apiVip != nil {
+		vips = append(vips, apiVip)
+	}
+	if ingressVip != nil {
+		vips = append(vips, ingressVip)
+	}
+	if dnsVip != nil {
+		vips = append(vips, dnsVip)
+	}
+	return getInterfaceAndNonVIPAddr(vips)
+}
+
+func GetIngressConfig(kubeconfigPath string) (ingressConfig IngressConfig, err error) {
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	if err != nil {
+		return ingressConfig, err
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return ingressConfig, err
+	}
+
+	nodes, err := clientset.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return ingressConfig, err
+	}
+
+	for _, node := range nodes.Items {
+		for _, address := range node.Status.Addresses {
+			if address.Type == v1.NodeInternalIP {
+				ingressConfig.Peers = append(ingressConfig.Peers, address.Address)
+			}
+		}
+	}
+
+	return ingressConfig, nil
+}
 
 func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, dnsVip net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) {
 	// Try cluster-config.yml first
@@ -205,12 +277,9 @@ func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip
 	if err != nil {
 		return node, err
 	}
-
-	vips := make([]net.IP, 0)
 	node.Cluster.APIVIPRecordType = "A"
 	node.Cluster.APIVIPEmptyType = "AAAA"
 	if apiVip != nil {
-		vips = append(vips, apiVip)
 		node.Cluster.APIVIP = apiVip.String()
 		if apiVip.To4() == nil {
 			node.Cluster.APIVIPRecordType = "AAAA"
@@ -220,23 +289,26 @@ func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip
 	node.Cluster.IngressVIPRecordType = "A"
 	node.Cluster.IngressVIPEmptyType = "AAAA"
 	if ingressVip != nil {
-		vips = append(vips, ingressVip)
 		node.Cluster.IngressVIP = ingressVip.String()
 		if ingressVip.To4() == nil {
 			node.Cluster.IngressVIPRecordType = "AAAA"
 			node.Cluster.IngressVIPEmptyType = "A"
 		}
 	}
 	if dnsVip != nil {
-		vips = append(vips, dnsVip)
 		node.Cluster.DNSVIP = dnsVip.String()
 	}
-	vipIface, nonVipAddr, err := getInterfaceAndNonVIPAddr(vips)
+	vipIface, nonVipAddr, err := GetVRRPConfig(apiVip, ingressVip, dnsVip)
 	if err != nil {
 		return node, err
 	}
 	node.NonVirtualIP = nonVipAddr.IP.String()
 
+	node.EnableUnicast = false
+	if os.Getenv("ENABLE_UNICAST") == "yes" {
+		node.EnableUnicast = true
+	}
+
 	resolvConfUpstreams, err := getDNSUpstreams(resolvConfPath)
 	if err != nil {
 		return node, err
@@ -261,9 +333,15 @@ func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip
 	return node, err
 }
 
+// getSortedBackends builds config to communicate with kube-api based on kubeconfigPath parameter value, if kubeconfigPath is not empty it will build the
+// config based on that content else config will point to localhost.
 func getSortedBackends(kubeconfigPath string) (backends []Backend, err error) {
 
-	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	kubeApiServerUrl := ""
+	if kubeconfigPath == "" {
+		kubeApiServerUrl = localhostKubeApiServerUrl
+	}
+	config, err := clientcmd.BuildConfigFromFlags(kubeApiServerUrl, kubeconfigPath)
 	if err != nil {
 		log.WithFields(logrus.Fields{
 			"err": err,
@@ -313,19 +391,24 @@ func GetLBConfig(kubeconfigPath string, apiPort, lbPort, statPort uint16, apiVip
 		LbPort:   lbPort,
 		StatPort: statPort,
 	}
+
 	// LB frontend address: IPv6 '::' , IPv4 ''
 	if apiVip.To4() == nil {
 		config.FrontendAddr = "::"
 	}
-
+	// Try reading master nodes details first from api-vip:kube-apiserver and failover to localhost:kube-apiserver
 	backends, err := getSortedBackends(kubeconfigPath)
 	if err != nil {
-		log.WithFields(logrus.Fields{
-			"kubeconfigPath": kubeconfigPath,
-		}).Error("Failed to retrieve API members information")
-		return config, err
+		log.Infof("An error occurred while trying to read master nodes details from api-vip:kube-apiserver: %v", err)
+		log.Infof("Trying to read master nodes details from localhost:kube-apiserver")
+		backends, err = getSortedBackends("")
+		if err != nil {
+			log.WithFields(logrus.Fields{
+				"kubeconfigPath": kubeconfigPath,
+			}).Error("Failed to retrieve API members information")
+			return config, err
+		}
 	}
-
 	// The backends port is the Etcd one, but we need to loadbalance the API one
 	for i := 0; i < len(backends); i++ {
 		backends[i].Port = apiPort