From 57b01b4ad210d1c3845867529c211ba1b2960e2b Mon Sep 17 00:00:00 2001 From: changluyi <47097611+changluyi@users.noreply.github.com> Date: Wed, 21 Jun 2023 10:17:14 +0800 Subject: [PATCH] ovn ic support dual (#2970) * ovn ic support dual --- .github/workflows/build-x86-image.yaml | 11 +++- Makefile | 74 +++++++++++++++++++++++++- dist/images/Dockerfile.base | 4 +- dist/images/start-ic-db.sh | 11 +++- pkg/controller/ovn-ic.go | 34 +++++++++--- pkg/util/const.go | 2 + pkg/util/net.go | 15 +++++- 7 files changed, 136 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 7bfceab7b91..3cbe0ee8527 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -979,6 +979,13 @@ jobs: - build-kube-ovn - build-e2e-binaries runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + ip-family: + - ipv4 + - ipv6 + - dual timeout-minutes: 15 steps: - uses: actions/checkout@v3 @@ -1043,12 +1050,12 @@ jobs: run: | sudo pip3 install j2cli sudo pip3 install "j2cli[yaml]" - sudo PATH=~/.local/bin:$PATH make kind-init-ovn-ic + sudo PATH=~/.local/bin:$PATH make kind-init-ovn-ic-${{ matrix.ip-family }} sudo cp -r /root/.kube/ ~/.kube/ sudo chown -R $(id -un). ~/.kube/ - name: Install Kube-OVN - run: make kind-install-ovn-ic + run: make kind-install-ovn-ic-${{ matrix.ip-family }} - name: Run E2E working-directory: ${{ env.E2E_DIR }} diff --git a/Makefile b/Makefile index 04736f98632..336d16d1b0e 100644 --- a/Makefile +++ b/Makefile @@ -328,10 +328,23 @@ kind-init-ipv4: kind-clean @$(MAKE) kind-create .PHONY: kind-init-ovn-ic -kind-init-ovn-ic: kind-clean-ovn-ic kind-init +kind-init-ovn-ic: kind-init-ovn-ic-ipv4 + +.PHONY: kind-init-ovn-ic-ipv4 +kind-init-ovn-ic-ipv4: kind-clean-ovn-ic kind-init @$(MAKE) kind-generate-config $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1) +.PHONY: kind-init-ovn-ic-ipv6 +kind-init-ovn-ic-ipv6: kind-clean-ovn-ic kind-init-ipv6 + @ip_family=ipv6 $(MAKE) kind-generate-config + $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1) + +.PHONY: kind-init-ovn-ic-dual +kind-init-ovn-ic-dual: kind-clean-ovn-ic kind-init-dual + @ip_family=dual $(MAKE) kind-generate-config + $(call kind_create_cluster,yamls/kind.yaml,kube-ovn1) + .PHONY: kind-init-ovn-submariner kind-init-ovn-submariner: kind-clean-ovn-submariner kind-init @pod_cidr_v4=10.18.0.0/16 svc_cidr_v4=10.112.0.0/12 $(MAKE) kind-generate-config @@ -439,7 +452,10 @@ kind-install-ipv4: kind-install-overlay-ipv4 kind-install-overlay-ipv4: kind-install .PHONY: kind-install-ovn-ic -kind-install-ovn-ic: kind-install +kind-install-ovn-ic: kind-install-ovn-ic-ipv4 + +.PHONY: kind-install-ovn-ic-ipv4 +kind-install-ovn-ic-ipv4: kind-install $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) kubectl config use-context kind-kube-ovn1 sed -e 's/10.16.0/10.18.0/g' \ @@ -461,6 +477,60 @@ kind-install-ovn-ic: kind-install sleep 6 docker exec ovn-ic-db ovn-ic-sbctl show +.PHONY: kind-install-ovn-ic-ipv6 +kind-install-ovn-ic-ipv6: kind-install-ipv6 + $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) + kubectl config use-context kind-kube-ovn1 + @$(MAKE) kind-untaint-control-plane + sed -e 's/fd00:10:16:/fd00:10:18:/g' \ + -e 's/fd00:10:96:/fd00:10:98:/g' \ + -e 's/fd00:100:64:/fd00:100:68:/g' \ + -e 's/VERSION=.*/VERSION=$(VERSION)/' \ + dist/images/install.sh | \ + IPV6=true bash + kubectl describe no + + docker run -d --name ovn-ic-db --network kind -e PROTOCOL="ipv6" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh + @set -e; \ + ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}"); \ + zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ + zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml + kubectl config use-context kind-kube-ovn + kubectl apply -f ovn-ic-0.yaml + kubectl config use-context kind-kube-ovn1 + kubectl apply -f ovn-ic-1.yaml + sleep 6 + docker exec ovn-ic-db ovn-ic-sbctl show + +.PHONY: kind-install-ovn-ic-dual +kind-install-ovn-ic-dual: kind-install-dual + $(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION)) + kubectl config use-context kind-kube-ovn1 + @$(MAKE) kind-untaint-control-plane + sed -e 's/10.16.0/10.18.0/g' \ + -e 's/10.96.0/10.98.0/g' \ + -e 's/100.64.0/100.68.0/g' \ + -e 's/fd00:10:16:/fd00:10:18:/g' \ + -e 's/fd00:10:96:/fd00:10:98:/g' \ + -e 's/fd00:100:64:/fd00:100:68:/g' \ + -e 's/VERSION=.*/VERSION=$(VERSION)/' \ + dist/images/install.sh | \ + DUAL_STACK=true bash + kubectl describe no + + docker run -d --name ovn-ic-db --network kind -e PROTOCOL="dual" $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh + @set -e; \ + + ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \ + zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \ + zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml + kubectl config use-context kind-kube-ovn + kubectl apply -f ovn-ic-0.yaml + kubectl config use-context kind-kube-ovn1 + kubectl apply -f ovn-ic-1.yaml + sleep 6 + docker exec ovn-ic-db ovn-ic-sbctl show + .PHONY: kind-install-ovn-submariner kind-install-ovn-submariner: kind-install $(call kind_load_submariner_images,kube-ovn) diff --git a/dist/images/Dockerfile.base b/dist/images/Dockerfile.base index 911c9e04cbf..0aa9a8daf6d 100644 --- a/dist/images/Dockerfile.base +++ b/dist/images/Dockerfile.base @@ -38,7 +38,9 @@ RUN cd /usr/src/ && git clone -b branch-22.12 --depth=1 https://github.com/ovn-o # fix reaching resubmit limit in underlay curl -s https://github.com/kubeovn/ovn/commit/6934f1a1eb5986a904eefb560c0d6d57811453d9.patch | git apply && \ # ovn-controller: do not send GARP on localnet for Kube-OVN ports - curl -s https://github.com/kubeovn/ovn/commit/8af8751cdb55f582c675db921f2526b06fd3d8c0.patch | git apply + curl -s https://github.com/kubeovn/ovn/commit/8af8751cdb55f582c675db921f2526b06fd3d8c0.patch | git apply && \ + # ovn-ic blacklist function not work on ipv6 + curl -s https://github.com/kubeovn/ovn/commit/78ab91005854532e7eb5c4fe6b2923ce292e3681.patch | git apply RUN apt install -y build-essential fakeroot \ autoconf automake bzip2 debhelper-compat dh-exec dh-python dh-sequence-python3 dh-sequence-sphinxdoc \ diff --git a/dist/images/start-ic-db.sh b/dist/images/start-ic-db.sh index 8dd16fd1eeb..fe46e843a88 100644 --- a/dist/images/start-ic-db.sh +++ b/dist/images/start-ic-db.sh @@ -2,7 +2,14 @@ set -eo pipefail TS_NAME=${TS_NAME:-ts} -TS_CIDR=${TS_CIDR:-169.254.100.0/24} +PROTOCOL=${PROTOCOL:-ipv4} +if [ "$PROTOCOL" = "ipv4" ]; then + TS_CIDR=${TS_CIDR:-169.254.100.0/24} +elif [ "$PROTOCOL" = "ipv6" ]; then + TS_CIDR=${TS_CIDR:-fe80:a9fe:64::/112} +elif [ "$PROTOCOL" = "dual" ]; then + TS_CIDR=${TS_CIDR:-"169.254.100.0/24,fe80:a9fe:64::/112"} +fi function quit { /usr/share/ovn/scripts/ovn-ctl stop_ic_ovsdb @@ -17,7 +24,7 @@ function gen_conn_str { trap quit EXIT if [[ -z "$NODE_IPS" && -z "$LOCAL_IP" ]]; then - /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes --db-ic-sb-create-insecure-remote=yes start_ic_ovsdb + /usr/share/ovn/scripts/ovn-ctl --db-ic-nb-create-insecure-remote=yes --db-ic-sb-create-insecure-remote=yes --db-ic-nb-addr="[::]" --db-ic-sb-addr="[::]" start_ic_ovsdb /usr/share/ovn/scripts/ovn-ctl status_ic_ovsdb ovn-ic-nbctl --may-exist ts-add "$TS_NAME" ovn-ic-nbctl set Transit_Switch ts external_ids:subnet="$TS_CIDR" diff --git a/pkg/controller/ovn-ic.go b/pkg/controller/ovn-ic.go index 18cc6589556..54661281584 100644 --- a/pkg/controller/ovn-ic.go +++ b/pkg/controller/ovn-ic.go @@ -17,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -253,14 +254,14 @@ func (c *Controller) establishInterConnection(config map[string]string) error { return err } - subnet, err := c.acquireLrpAddress(util.InterconnectionSwitch) + lrpIP, err := c.acquireLrpAddress(util.InterconnectionSwitch) if err != nil { klog.Errorf("failed to acquire lrp address, %v", err) return err } lrpName := fmt.Sprintf("%s-ts", config["az-name"]) - if err := c.ovnClient.CreateLogicalPatchPort(util.InterconnectionSwitch, c.config.ClusterRouter, tsPort, lrpName, subnet, util.GenerateMac(), chassises...); err != nil { + if err := c.ovnClient.CreateLogicalPatchPort(util.InterconnectionSwitch, c.config.ClusterRouter, tsPort, lrpName, lrpIP, util.GenerateMac(), chassises...); err != nil { klog.Errorf("failed to create ovn-ic lrp %v", err) return err } @@ -281,8 +282,17 @@ func (c *Controller) acquireLrpAddress(ts string) (string, error) { } for { - random := util.GenerateRandomV4IP(cidr) + var random string + var ips []string + v4Cidr, v6Cidr := util.SplitStringIP(cidr) + if v4Cidr != "" { + ips = append(ips, util.GenerateRandomV4IP(v4Cidr)) + } + if v6Cidr != "" { + ips = append(ips, util.GenerateRandomV6IP(v6Cidr)) + } + random = strings.Join(ips, ",") // find a free address if !existAddress.Has(random) { return random, nil @@ -420,6 +430,8 @@ func stripPrefix(policyMatch string) (string, error) { matches := strings.Split(policyMatch, "==") if strings.Trim(matches[0], " ") == util.MatchV4Dst { return strings.Trim(matches[1], " "), nil + } else if strings.Trim(matches[0], " ") == util.MatchV6Dst { + return strings.Trim(matches[1], " "), nil } else { return "", fmt.Errorf("policy %s is mismatched", policyMatch) } @@ -464,9 +476,19 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) { if _, ok := policyMap[lrRoute.IPPrefix]; ok { delete(policyMap, lrRoute.IPPrefix) } else { - matchFiled := util.MatchV4Dst + " == " + lrRoute.IPPrefix - if err := c.ovnClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, matchFiled, ovnnb.LogicalRouterPolicyActionAllow, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil { - klog.Errorf("adding router policy failed %v", err) + var matchFiled string + if util.CheckProtocol(lrRoute.IPPrefix) == kubeovnv1.ProtocolIPv4 { + matchFiled = util.MatchV4Dst + " == " + lrRoute.IPPrefix + if err := c.ovnClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, matchFiled, ovnnb.LogicalRouterPolicyActionAllow, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil { + klog.Errorf("adding router policy failed %v", err) + } + } + + if util.CheckProtocol(lrRoute.IPPrefix) == kubeovnv1.ProtocolIPv6 { + matchFiled = util.MatchV6Dst + " == " + lrRoute.IPPrefix + if err := c.ovnClient.AddLogicalRouterPolicy(lr.Name, util.OvnICPolicyPriority, matchFiled, ovnnb.LogicalRouterPolicyActionAllow, nil, map[string]string{key: value, "vendor": util.CniTypeName}); err != nil { + klog.Errorf("adding router policy failed %v", err) + } } } } diff --git a/pkg/util/const.go b/pkg/util/const.go index 68be2f02f00..a38198b3516 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -236,6 +236,8 @@ const ( MatchV4Src = "ip4.src" MatchV4Dst = "ip4.dst" + MatchV6Src = "ip6.src" + MatchV6Dst = "ip6.dst" U2OInterconnName = "u2o-interconnection.%s.%s" U2OExcludeIPAg = "%s.u2o_exclude_ip.%s" diff --git a/pkg/util/net.go b/pkg/util/net.go index 521a84d3355..78d50b56464 100644 --- a/pkg/util/net.go +++ b/pkg/util/net.go @@ -181,13 +181,24 @@ func AddressCount(network *net.IPNet) float64 { } func GenerateRandomV4IP(cidr string) string { + return genRandomIP(cidr, false) +} + +func GenerateRandomV6IP(cidr string) string { + return genRandomIP(cidr, true) +} + +func genRandomIP(cidr string, isIPv6 bool) string { if len(strings.Split(cidr, "/")) != 2 { return "" } ip := strings.Split(cidr, "/")[0] netMask, _ := strconv.Atoi(strings.Split(cidr, "/")[1]) - hostNum := 32 - netMask - add, err := rand.Int(rand.Reader, big.NewInt(1<<(uint(hostNum)-1))) + hostBits := 32 - netMask + if isIPv6 { + hostBits = 128 - netMask + } + add, err := rand.Int(rand.Reader, big.NewInt(1<<(uint(hostBits)-1))) if err != nil { LogFatalAndExit(err, "failed to generate random ip") }