Skip to content

Commit

Permalink
support specifying routes when providing IPAM for other CNI plugins (#…
Browse files Browse the repository at this point in the history 
…3904)

Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>
  • Loading branch information
@zhangzujian
zhangzujian committed Apr 11, 2024
1 parent 96a5695 commit b32f607
Show file tree
Hide file tree
Showing 13 changed files with 723 additions and 67 deletions.
119 changes: 119 additions & 0 deletions .github/workflows/build-x86-image.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1263,6 +1263,124 @@ jobs:
name: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log
path: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

multus-conformance-e2e:
name: Multus Conformance E2E
needs:
- build-kube-ovn
- build-e2e-binaries
runs-on: ubuntu-22.04
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
ip-family:
- ipv4
- ipv6
- dual
steps:
- uses: jlumbroso/free-disk-space@v1.3.1
with:
android: true
dotnet: true
haskell: true
docker-images: false
large-packages: false
tool-cache: false
swap-storage: false

- uses: actions/checkout@v4

- name: Create the default branch directory
if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
run: mkdir -p test/e2e/source

- name: Check out the default branch
if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
uses: actions/checkout@v4
with:
ref: ${{ github.event.repository.default_branch }}
fetch-depth: 1
path: test/e2e/source

- name: Export E2E directory
run: |
if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
echo "E2E_DIR=." >> "$GITHUB_ENV"
else
echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
fi
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION || '' }}
go-version-file: ${{ env.E2E_DIR }}/go.mod
check-latest: true
cache: false

- name: Export Go full version
run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"

- name: Go cache
uses: actions/cache/restore@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-

- name: Install kind
uses: helm/kind-action@v1.9.0
with:
version: ${{ env.KIND_VERSION }}
install_only: true

- name: Install ginkgo
working-directory: ${{ env.E2E_DIR }}
run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo

- name: Download kube-ovn image
uses: actions/download-artifact@v4
with:
name: kube-ovn

- name: Load images
run: docker load -i kube-ovn.tar

- name: Create kind cluster
run: |
sudo pip3 install j2cli
sudo pip3 install "j2cli[yaml]"
sudo PATH=~/.local/bin:$PATH make kind-init-${{ matrix.ip-family }}
sudo cp -r /root/.kube/ ~/.kube/
sudo chown -R $(id -un). ~/.kube/
- name: Install Kube-OVN
run: make kind-install-${{ matrix.ip-family }}

- name: Install Multus
run: make kind-install-multus

- name: Run E2E
working-directory: ${{ env.E2E_DIR }}
env:
E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
E2E_IP_FAMILY: ${{ matrix.ip-family }}
run: make kube-ovn-multus-conformance-e2e

- name: kubectl ko log
if: failure()
run: |
make kubectl-ko-log
mv kubectl-ko-log.tar.gz multus-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
- name: upload kubectl ko log
uses: actions/upload-artifact@v4
if: failure()
with:
name: multus-conformance-e2e-${{ matrix.ip-family }}-ko-log
path: multus-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

chart-test:
name: Chart Installation/Uninstallation Test
needs: build-kube-ovn
Expand Down Expand Up @@ -2377,6 +2495,7 @@ jobs:
- cyclonus-netpol-e2e
- kube-ovn-conformance-e2e
- kube-ovn-ic-conformance-e2e
- multus-conformance-e2e
- ovn-vpc-nat-gw-conformance-e2e
- iptables-vpc-nat-gw-conformance-e2e
- webhook-e2e
Expand Down
10 changes: 10 additions & 0 deletions Makefile.e2e
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ e2e-build:
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/k8s-network
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/kube-ovn
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-ic
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw
Expand Down Expand Up @@ -130,6 +131,15 @@ kube-ovn-submariner-conformance-e2e:
--context kind-kube-ovn --tocontext kind-kube-ovn1 \
--verbose --disruptive-tests

.PHONY: kube-ovn-multus-conformance-e2e
kube-ovn-multus-conformance-e2e:
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus
E2E_BRANCH=$(E2E_BRANCH) \
E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
--focus=CNI:Kube-OVN ./test/e2e/multus/multus.test -- $(TEST_BIN_ARGS)

.PHONY: kube-ovn-lb-svc-conformance-e2e
kube-ovn-lb-svc-conformance-e2e:
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc
Expand Down
41 changes: 29 additions & 12 deletions cmd/cni/cni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/containernetworking/cni/pkg/version"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/netconf"
"github.com/kubeovn/kube-ovn/pkg/request"
"github.com/kubeovn/kube-ovn/pkg/util"
"github.com/kubeovn/kube-ovn/versions"
Expand Down Expand Up @@ -73,6 +74,7 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R
result := current.Result{
CNIVersion: current.ImplementedSpecVersion,
DNS: cniResponse.DNS,
Routes: parseRoutes(cniResponse.Routes),
}
_, mask, _ := net.ParseCIDR(cniResponse.CIDR)
podIface := current.Interface{
Expand All @@ -84,20 +86,21 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R
case kubeovnv1.ProtocolIPv4:
ip, route := assignV4Address(cniResponse.IPAddress, cniResponse.Gateway, mask)
result.IPs = []*current.IPConfig{ip}
if route != nil {
if len(result.Routes) == 0 && route != nil {
result.Routes = []*types.Route{route}
}
result.Interfaces = []*current.Interface{&podIface}
case kubeovnv1.ProtocolIPv6:
ip, route := assignV6Address(cniResponse.IPAddress, cniResponse.Gateway, mask)
result.IPs = []*current.IPConfig{ip}
if route != nil {
if len(result.Routes) == 0 && route != nil {
result.Routes = []*types.Route{route}
}
result.Interfaces = []*current.Interface{&podIface}
case kubeovnv1.ProtocolDual:
var netMask *net.IPNet
var gwStr string
addRoutes := len(result.Routes) == 0
for _, cidrBlock := range strings.Split(cniResponse.CIDR, ",") {
_, netMask, _ = net.ParseCIDR(cidrBlock)
gwStr = ""
Expand All @@ -109,7 +112,7 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R

ip, route := assignV4Address(ipStr, gwStr, netMask)
result.IPs = append(result.IPs, ip)
if route != nil {
if addRoutes && route != nil {
result.Routes = append(result.Routes, route)
}
} else if util.CheckProtocol(cidrBlock) == kubeovnv1.ProtocolIPv6 {
Expand All @@ -120,7 +123,7 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R

ip, route := assignV6Address(ipStr, gwStr, netMask)
result.IPs = append(result.IPs, ip)
if route != nil {
if addRoutes && route != nil {
result.Routes = append(result.Routes, route)
}
}
Expand All @@ -131,6 +134,24 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R
return result
}

func parseRoutes(routes []request.Route) []*types.Route {
parsedRoutes := make([]*types.Route, len(routes))
for i, r := range routes {
if r.Destination == "" {
if util.CheckProtocol(r.Gateway) == kubeovnv1.ProtocolIPv4 {
r.Destination = "0.0.0.0/0"
} else {
r.Destination = "::/0"
}
}
parsedRoutes[i] = &types.Route{GW: net.ParseIP(r.Gateway)}
if _, cidr, err := net.ParseCIDR(r.Destination); err == nil {
parsedRoutes[i].Dst = *cidr
}
}
return parsedRoutes
}

func cmdDel(args *skel.CmdArgs) error {
netConf, _, err := loadNetConf(args.StdinData)
if err != nil {
Expand Down Expand Up @@ -167,20 +188,16 @@ func cmdDel(args *skel.CmdArgs) error {
return nil
}

type ipamConf struct {
ServerSocket string `json:"server_socket"`
Provider string `json:"provider"`
}

func loadNetConf(bytes []byte) (*netConf, string, error) {
n := &netConf{}
func loadNetConf(bytes []byte) (*netconf.NetConf, string, error) {
n := &netconf.NetConf{}
if err := json.Unmarshal(bytes, n); err != nil {
return nil, "", types.NewError(types.ErrDecodingFailure, "failed to load netconf", err.Error())
}

if n.Type != util.CniTypeName && n.IPAM != nil {
n.Provider = n.IPAM.Provider
n.ServerSocket = n.IPAM.ServerSocket
n.Routes = n.IPAM.Routes
}

if n.ServerSocket == "" {
Expand All @@ -191,7 +208,7 @@ func loadNetConf(bytes []byte) (*netConf, string, error) {
n.Provider = util.OvnProvider
}

n.postLoad()
n.PostLoad()
return n, n.CNIVersion, nil
}

Expand Down
33 changes: 30 additions & 3 deletions pkg/daemon/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"strconv"
"strings"
Expand Down Expand Up @@ -211,6 +212,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
return
}

routes = append(podRequest.Routes, routes...)
if strings.HasSuffix(podRequest.Provider, util.OvnProvider) && subnet != "" {
podSubnet, err := csh.Controller.subnetsLister.Get(subnet)
if err != nil {
Expand Down Expand Up @@ -284,17 +286,17 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
}
}

routes = append(podRequest.Routes, routes...)
macAddr = pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podRequest.Provider)]
klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s, custom routes %v", ifName, macAddr, ipAddr, cidr, gw, routes)
podNicName = ifName
switch nicType {
case util.InternalType:
podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP)
podNicName, routes, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP)
case util.DpdkType:
err = csh.configureDpdkNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, ingress, egress, getShortSharedDir(pod.UID, podRequest.VhostUserSocketVolumeName), podRequest.VhostUserSocketName)
routes = nil
default:
err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP)
routes, err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP)
}
if err != nil {
errMsg := fmt.Errorf("configure nic failed %v", err)
Expand All @@ -319,6 +321,30 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
}
return
}
} else if len(routes) != 0 {
hasDefaultRoute := make(map[string]bool, 2)
for _, r := range routes {
if r.Destination == "" {
hasDefaultRoute[util.CheckProtocol(r.Gateway)] = true
continue
}
if _, cidr, err := net.ParseCIDR(r.Destination); err == nil {
if ones, _ := cidr.Mask.Size(); ones == 0 {
hasDefaultRoute[util.CheckProtocol(r.Gateway)] = true
}
}
}
if len(hasDefaultRoute) != 0 {
// remove existing default route so other CNI plugins, such as macvlan, can add the new default route correctly
if err = csh.removeDefaultRoute(podRequest.NetNs, hasDefaultRoute[kubeovnv1.ProtocolIPv4], hasDefaultRoute[kubeovnv1.ProtocolIPv6]); err != nil {
errMsg := fmt.Errorf("failed to remove existing default route for interface %s of pod %s/%s: %v", podRequest.IfName, podRequest.PodNamespace, podRequest.PodName, err)
klog.Error(errMsg)
if err = resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()}); err != nil {
klog.Errorf("failed to write response: %v", err)
}
return
}
}
}

response := &request.CniResponse{
Expand All @@ -327,6 +353,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
MacAddress: macAddr,
CIDR: cidr,
PodNicName: podNicName,
Routes: routes,
}
if isDefaultRoute {
response.Gateway = gw
Expand Down
Loading

0 comments on commit b32f607

Please sign in to comment.