Skip to content

Commit

Permalink
feat: add simple gateway implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
oilbeater committed Mar 19, 2019
1 parent fd59a2c commit d37264e
Show file tree
Hide file tree
Showing 8 changed files with 423 additions and 10 deletions.
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ build-go:
CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn -ldflags "-w -s" -v ./cmd/cni
CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn-controller -ldflags "-w -s" -v ./cmd/controller
CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn-daemon -ldflags "-w -s" -v ./cmd/daemon
CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn-gateway -ldflags "-w -s" -v ./cmd/gateway

release: build-go
docker build -t index.alauda.cn/alaudak8s/kube-ovn-node:`cat VERSION` -f dist/images/Dockerfile.node dist/images/
Expand Down
59 changes: 59 additions & 0 deletions cmd/gateway/gateway.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package main

import (
"bitbucket.org/mathildetech/kube-ovn/pkg/gateway"
"bitbucket.org/mathildetech/kube-ovn/pkg/ovs"
"bitbucket.org/mathildetech/kube-ovn/pkg/util"
"github.com/vishvananda/netlink"
"k8s.io/klog"
"os"
)

func main() {
klog.SetOutput(os.Stdout)
defer klog.Flush()

config, err := gateway.ParseFlags()
if err != nil {
klog.Errorf("parse config failed %v", err)
os.Exit(1)
}

bridge, err := util.NicToBridge(config.Interface)
if err != nil {
klog.Errorf("failed to move nic to bridge %v", err)
os.Exit(1)
}
klog.Infof("create bridge %v for gw", bridge)

ovnClient := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.OvnSbHost, config.OvnSbPort, "", "", "")

err = ovnClient.CreateGatewayRouter(config.EdgeRouterName, config.Chassis)
if err != nil {
klog.Errorf("failed to crate gateway router %v", err)
os.Exit(1)
}

err = ovnClient.CreateTransitLogicalSwitch(config.TransitSwitchName, config.ClusterRouterName, config.EdgeRouterName, config.ClusterRouterIP, config.EdgeRouterIP)
if err != nil {
klog.Errorf("failed to connect edge and cluster by transit %v", err)
os.Exit(1)
}

addrList, err := netlink.AddrList(bridge, netlink.FAMILY_V4)
if err != nil {
klog.Errorf("failed to list %s addr %v", bridge, err)
os.Exit(1)
}
if len(addrList) == 0 {
klog.Errorf("nic %s has no ip address", bridge)
os.Exit(1)
}

err = ovnClient.CreateOutsideLogicalSwitch(config.OutsideSwitchName, config.EdgeRouterName, addrList[0].IPNet.String(), bridge.Attrs().HardwareAddr.String())
if err != nil {
klog.Errorf("failed to connect edge to outside %v", err)
os.Exit(1)
}
return
}
7 changes: 5 additions & 2 deletions dist/images/Dockerfile.cni
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ RUN yum install -y \
unbound unbound-devel && \
yum clean all

RUN rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-2.10.1-1.el7.centos.x86_64.rpm
RUN rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-2.10.1-1.el7.centos.x86_64.rpm && \
rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-ovn-common-2.10.1-1.el7.centos.x86_64.rpm && \
rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-ovn-host-2.10.1-1.el7.centos.x86_64.rpm

COPY start-cniserver.sh /kube-ovn/start-cniserver.sh
COPY kube-ovn.conflist /kube-ovn/kube-ovn.conflist
Expand All @@ -20,4 +22,5 @@ WORKDIR /kube-ovn
CMD ["sh", "start-cniserver.sh"]

COPY kube-ovn /kube-ovn/kube-ovn
COPY kube-ovn-daemon /kube-ovn/kube-ovn-daemon
COPY kube-ovn-daemon /kube-ovn/kube-ovn-daemon
COPY kube-ovn-gateway /kube-ovn/kube-ovn-gateway
2 changes: 1 addition & 1 deletion pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func NewController(

controller := &Controller{
config: config,
ovnClient: ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer),
ovnClient: ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer),
kubeclientset: config.KubeClient,

podsLister: podInformer.Lister(),
Expand Down
8 changes: 4 additions & 4 deletions pkg/controller/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func InitDefaultLogicalSwitch(config *Configuration) error {
}

func InitNodeSwitch(config *Configuration) error {
client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer)
client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer)
ss, err := client.ListLogicalSwitch()
if err != nil {
return err
Expand All @@ -70,7 +70,7 @@ func InitNodeSwitch(config *Configuration) error {
}

func InitClusterRouter(config *Configuration) error {
client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer)
client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer)
lrs, err := client.ListLogicalRouter()
if err != nil {
return err
Expand All @@ -85,7 +85,7 @@ func InitClusterRouter(config *Configuration) error {
}

func InitLoadBalancer(config *Configuration) error {
client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer)
client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer)
tcpLb, err := client.FindLoadbalancer(config.ClusterTcpLoadBalancer)
if err != nil {
return fmt.Errorf("failed to find tcp lb %v", err)
Expand Down Expand Up @@ -121,7 +121,7 @@ func InitLoadBalancer(config *Configuration) error {
}

func InitDnsTable(config *Configuration) error {
client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer)
client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer)
uuid, err := client.CreateDnsTable()
if err != nil {
return err
Expand Down
61 changes: 61 additions & 0 deletions pkg/gateway/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package gateway

import (
"flag"
"github.com/spf13/pflag"
)

type Configuration struct {
Interface string
SnatIP string
ClusterRouterIP string
EdgeRouterName string
EdgeRouterIP string
Chassis string
TransitSwitchName string
OutsideSwitchName string
ClusterRouterName string
OvnNbHost string
OvnNbPort int
OvnSbHost string
OvnSbPort int
}

func ParseFlags() (*Configuration, error) {
var (
argInterface = pflag.String("interface", "eth0", "The gateway interface")
argSnatIP = pflag.String("snat-ip", "", "The snat ip")
argClusterRouterIP = pflag.String("cluster-router-ip", "172.16.255.2/30", "The cluster route to transit switch ip")
argEdgeRouterIP = pflag.String("edge-router-ip", "172.16.255.1/30", "The edge router to transit switch ip")
argChassis = pflag.String("chassis", "", "chassis id found in ovn-sb")
argTransitSwitchName = pflag.String("transit-switch-name", "transit", "The name of switch between cluster router and edge router.")
argOutsideSwitchName = pflag.String("outside-switch-name", "outside", "The name of switch between edge router and physic interface.")
argEdgeRouterName = pflag.String("edge-router-name", "edge", "The edge router name")
argClusterRouterName = pflag.String("cluster-router-name", "ovn-cluster", "The cluster router name")
argOvnNbHost = pflag.String("ovn-nb-host", "", "")
argOvnNbPort = pflag.Int("ovn-nb-port", 6641, "")
argOvnSbHost = pflag.String("ovn-sb-host", "", "")
argOvnSbPort = pflag.Int("ovn-sb-port", 6642, "")
)
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
pflag.Parse()
flag.CommandLine.Parse(make([]string, 0))

config := &Configuration{
Interface: *argInterface,
SnatIP: *argSnatIP,
ClusterRouterIP: *argClusterRouterIP,
EdgeRouterIP: *argEdgeRouterIP,
EdgeRouterName: *argEdgeRouterName,
Chassis: *argChassis,
TransitSwitchName: *argTransitSwitchName,
ClusterRouterName: *argClusterRouterName,
OutsideSwitchName: *argOutsideSwitchName,
OvnNbHost: *argOvnNbHost,
OvnNbPort: *argOvnNbPort,
OvnSbHost: *argOvnSbHost,
OvnSbPort: *argOvnSbPort,
}

return config, nil
}
83 changes: 80 additions & 3 deletions pkg/ovs/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

type Client struct {
OvnNbAddress string
OvnSbAddress string
ClusterRouter string
ClusterTcpLoadBalancer string
ClusterUdpLoadBalancer string
Expand All @@ -32,7 +33,7 @@ func (c Client) ovnCommand(arg ...string) (string, error) {
cmdArgs = append(cmdArgs, arg...)
raw, err := exec.Command(OvnNbCtl, cmdArgs...).CombinedOutput()
if err != nil {
return "", fmt.Errorf(string(raw))
return "", fmt.Errorf("%s, %v", string(raw), err)
}
return trimCommandOutput(raw), nil
}
Expand All @@ -42,10 +43,11 @@ func trimCommandOutput(raw []byte) string {
return strings.Trim(output, "\"")
}

func NewClient(ovnNbHost string, ovnNbPort int, clusterRouter, ClusterTcpLoadBalancer, ClusterUdpLoadBalancer string) *Client {
func NewClient(ovnNbHost string, ovnNbPort int, ovnSbHost string, ovnSbPort int, ClusterRouter, ClusterTcpLoadBalancer, ClusterUdpLoadBalancer string) *Client {
return &Client{
OvnNbAddress: fmt.Sprintf("tcp:%s:%d", ovnNbHost, ovnNbPort),
ClusterRouter: clusterRouter,
OvnSbAddress: fmt.Sprintf("tcp:%s:%d", ovnSbHost, ovnSbPort),
ClusterRouter: ClusterRouter,
ClusterTcpLoadBalancer: ClusterTcpLoadBalancer,
ClusterUdpLoadBalancer: ClusterUdpLoadBalancer,
}
Expand Down Expand Up @@ -114,6 +116,75 @@ type Nic struct {
Gateway string
}

func (c Client) CreateTransitLogicalSwitch(ls, clusterLr, edgeLr, toClusterIP, toEdgeIP string) error {
mac := util.GenerateMac()
edgeToTransit := fmt.Sprintf("%s-%s", edgeLr, ls)
transitToEdge := fmt.Sprintf("%s-%s", ls, edgeLr)
_, err := c.ovnCommand(WaitSb, MayExist, "ls-add", ls, "--",
"lrp-add", edgeLr, edgeToTransit, mac, toEdgeIP, "--",
"lsp-add", ls, transitToEdge, "--",
"lsp-set-type", transitToEdge, "router", "--",
"lsp-set-addresses", transitToEdge, mac, "--",
"lsp-set-options", transitToEdge, fmt.Sprintf("router-port=%s", edgeToTransit))
if err != nil {
klog.Errorf("connect edge to transit failed %v", err)
return err
}

mac = util.GenerateMac()
clusterToTransit := fmt.Sprintf("%s-%s", clusterLr, ls)
transitToCluster := fmt.Sprintf("%s-%s", ls, clusterLr)
_, err = c.ovnCommand("lrp-add", clusterLr, clusterToTransit, mac, toClusterIP, "--",
"lsp-add", ls, transitToCluster, "--",
"lsp-set-type", transitToCluster, "router", "--",
"lsp-set-addresses", transitToCluster, mac, "--",
"lsp-set-options", transitToCluster, fmt.Sprintf("router-port=%s", clusterToTransit))
if err != nil {
klog.Errorf("connect cluster to transit failed %v", err)
return err
}
return nil
}

func (c Client) CreateOutsideLogicalSwitch(ls, edgeLr, ip, mac string) error {
// 1. create outside logical switch
_, err := c.ovnCommand(WaitSb, MayExist, "ls-add", ls)
if err != nil {
klog.Errorf("create outside ls %s failed, %v", ls, err)
return err
}

// 2. connect outside ls with edge lr
outsideToEdge := fmt.Sprintf("%s-%s", ls, edgeLr)
edgeToOutside := fmt.Sprintf("%s-%s", edgeLr, ls)
_, err = c.ovnCommand("lrp-add", edgeLr, edgeToOutside, mac, ip)
if err != nil {
klog.Errorf("create lsp on edge failed, %v", err)
return err
}

_, err = c.ovnCommand(WaitSb, MayExist, "lsp-add", ls, outsideToEdge, "--",
"lsp-set-type", outsideToEdge, "router", "--",
"lsp-set-addresses", outsideToEdge, mac, "--",
"lsp-set-options", outsideToEdge, fmt.Sprintf("router-port=%s", edgeToOutside))
if err != nil {
klog.Errorf("failed to connect outside to edge, %v", err)
return err
}

// 3. create localnet port to connect outside to physic net
outsideToLocal := fmt.Sprintf("%s-localnet", ls)
_, err = c.ovnCommand(WaitSb, MayExist, "lsp-add", ls, outsideToLocal, "--",
"lsp-set-addresses", outsideToLocal, "unknown", "--",
"lsp-set-type", outsideToLocal, "localnet", "--",
"lsp-set-options", outsideToLocal, "network_name=dataNet")
if err != nil {
klog.Errorf("failed to create localnet port %v", err)
return err
}
return nil
}

func (c Client) CreateLogicalSwitch(ls, subnet, gateway, excludeIps string) error {
_, err := c.ovnCommand(WaitSb, MayExist, "ls-add", ls, "--",
"set", "logical_switch", ls, fmt.Sprintf("other_config:subnet=%s", subnet), "--",
Expand Down Expand Up @@ -207,6 +278,12 @@ func (c Client) DeleteLogicalSwitch(ls string) error {
return nil
}

func (c Client) CreateGatewayRouter(lr, chassis string) error {
_, err := c.ovnCommand(WaitSb, MayExist, "lr-add", lr, "--",
"set", "logical_router", lr, fmt.Sprintf("options:chassis=%s", chassis))
return err
}

func (c Client) CreateLogicalRouter(lr string) error {
_, err := c.ovnCommand(WaitSb, MayExist, "lr-add", lr)
return err
Expand Down

0 comments on commit d37264e

Please sign in to comment.