Skip to content

Commit

Permalink
Ovn ic ecmp enhance 1.11 (#3609)
Browse files Browse the repository at this point in the history
* ovn support ic ecmp enhance
  • Loading branch information
changluyi committed Jan 16, 2024
1 parent 7a5c4aa commit 7b863fb
Show file tree
Hide file tree
Showing 27 changed files with 1,564 additions and 265 deletions.
31 changes: 18 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -252,8 +252,9 @@ kind-init-ipv4: kind-clean
@$(MAKE) kind-create

.PHONY: kind-init-ovn-ic
kind-init-ovn-ic: kind-clean-ovn-ic kind-init
@$(MAKE) kind-generate-config
kind-init-ovn-ic: kind-clean-ovn-ic
@ha=true $(MAKE) kind-init
@ovn_ic=true $(MAKE) kind-generate-config
$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)

.PHONY: kind-init-iptables
Expand Down Expand Up @@ -311,7 +312,8 @@ kind-install-chart: kind-load-image kind-untaint-control-plane
helm install kubeovn ./kubeovn-helm \
--set global.images.kubeovn.tag=$(VERSION) \
--set replicaCount=$$(echo $$ips | awk -F ',' '{print NF}') \
--set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')"
--set MASTER_NODES="$$(echo $$ips | sed 's/,/\\,/g')" \
--set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)
sleep 60
kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central
kubectl -n kube-system rollout status --timeout=1s deployment/kube-ovn-controller
Expand All @@ -324,7 +326,8 @@ kind-upgrade-chart: kind-load-image
helm upgrade kubeovn ./kubeovn-helm \
--set global.images.kubeovn.tag=$(VERSION) \
--set replicaCount=$$(echo $(OVN_DB_IPS) | awk -F ',' '{print NF}') \
--set MASTER_NODES='$(OVN_DB_IPS)'
--set MASTER_NODES='$(OVN_DB_IPS)' \
--set func.ENABLE_IC=$$(kubectl get node --show-labels | grep -q "ovn.kubernetes.io/ic-gw" && echo true || echo false)
sleep 90
kubectl -n kube-system rollout status --timeout=1s deployment/ovn-central
kubectl -n kube-system rollout status --timeout=1s deployment/kube-ovn-controller
Expand Down Expand Up @@ -357,27 +360,29 @@ kind-install-ipv4: kind-install-overlay-ipv4
kind-install-overlay-ipv4: kind-install

.PHONY: kind-install-ovn-ic
kind-install-ovn-ic: kind-install
kind-install-ovn-ic:
@ENABLE_IC=true $(MAKE) kind-install
$(call kind_load_image,kube-ovn1,$(REGISTRY)/kube-ovn:$(VERSION))
kubectl config use-context kind-kube-ovn1
$(MAKE) kind-untaint-control-plane
sed -e 's/10.16.0/10.18.0/g' \
-e 's/10.96.0/10.98.0/g' \
-e 's/100.64.0/100.68.0/g' \
-e 's/VERSION=.*/VERSION=$(VERSION)/' \
dist/images/install.sh | bash
dist/images/install.sh | ENABLE_IC=true bash
kubectl describe no

docker run -d --name ovn-ic-db --network kind $(REGISTRY)/kube-ovn:$(VERSION) bash start-ic-db.sh
@set -e; \
ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \
zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-worker j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
kubectl config use-context kind-kube-ovn
sed 's/VERSION=.*/VERSION=$(VERSION)/' dist/images/install-ic-server.sh | bash

@set -e; \
ic_db_host=$$(kubectl get deployment ovn-ic-server -n kube-system -o jsonpath='{range .spec.template.spec.containers[0].env[?(@.name=="NODE_IPS")]}{.value}{end}'); \
ic_db_host=$${ic_db_host%?}; \
zone=az0 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn-worker,kube-ovn-worker2,kube-ovn-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
zone=az1 ic_db_host=$$ic_db_host gateway_node_name='kube-ovn1-worker,kube-ovn1-worker2,kube-ovn1-control-plane' j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
kubectl apply -f ovn-ic-0.yaml
kubectl config use-context kind-kube-ovn1
kubectl apply -f ovn-ic-1.yaml
sleep 6
docker exec ovn-ic-db ovn-ic-sbctl show

.PHONY: kind-install-underlay
kind-install-underlay: kind-install-underlay-ipv4
Expand Down
4 changes: 4 additions & 0 deletions cmd/cmdmain.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/kubeovn/kube-ovn/cmd/controller"
"github.com/kubeovn/kube-ovn/cmd/controller_health_check"
"github.com/kubeovn/kube-ovn/cmd/daemon"
"github.com/kubeovn/kube-ovn/cmd/ovn_ic_controller"
"github.com/kubeovn/kube-ovn/cmd/ovn_leader_checker"
"github.com/kubeovn/kube-ovn/cmd/ovn_monitor"
"github.com/kubeovn/kube-ovn/cmd/pinger"
Expand All @@ -24,6 +25,7 @@ const (
CmdSpeaker = "kube-ovn-speaker"
CmdControllerHealthCheck = "kube-ovn-controller-healthcheck"
CmdOvnLeaderChecker = "kube-ovn-leader-checker"
CmdOvnICController = "kube-ovn-ic-controller"
)

func main() {
Expand All @@ -45,6 +47,8 @@ func main() {
controller_health_check.CmdMain()
case CmdOvnLeaderChecker:
ovn_leader_checker.CmdMain()
case CmdOvnICController:
ovn_ic_controller.CmdMain()
default:
util.LogFatalAndExit(nil, "%s is an unknown command", cmd)
}
Expand Down
24 changes: 24 additions & 0 deletions cmd/ovn_ic_controller/ovn_ic_controller.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package ovn_ic_controller

import (
"k8s.io/klog/v2"
"k8s.io/sample-controller/pkg/signals"

"github.com/kubeovn/kube-ovn/pkg/ovn_ic_controller"
"github.com/kubeovn/kube-ovn/pkg/util"
"github.com/kubeovn/kube-ovn/versions"
)

func CmdMain() {
defer klog.Flush()

klog.Infof(versions.String())
config, err := ovn_ic_controller.ParseFlags()
if err != nil {
util.LogFatalAndExit(err, "failed to parse config")
}

stopCh := signals.SetupSignalHandler()
ctl := ovn_ic_controller.NewController(config)
ctl.Run(stopCh)
}
3 changes: 2 additions & 1 deletion dist/images/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn && \
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker && \
ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-ic-controller
47 changes: 47 additions & 0 deletions dist/images/clean-ic-az-db.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/bin/bash

if [ "$#" -ne 2 ]; then
echo "use method $0 {az|node} {azName|nodeName}"
echo " eg: ./clean-ic-az-db.sh az az1"
echo " eg: ./clean-ic-az-db.sh node kube-ovn-worker; it will delete all resource of az that the node belong to"
exit 1
fi

filter_type=$1
filter_value=$2
availability_zone_uuid=

if [ "$filter_type" != "az" ] && [ "$filter_type" != "node" ]; then
echo "filter_type should be az or node."
exit 1
fi

if [ "$filter_type" == "az" ]; then
availability_zone_uuid=$(ovn-ic-sbctl --columns=_uuid find availability_zone name=$filter_value | awk '{print $3}')
fi

echo $availability_zone_uuid

if [ "$filter_type" == "node" ]; then
availability_zone_uuid=$(ovn-ic-sbctl --columns=availability_zone find gateway hostname=$filter_value | awk '{print $3}')
fi

if ! ovn-ic-sbctl get availability_zone $availability_zone_uuid name >/dev/null 2>&1; then
echo "Availability zone $availability_zone_uuid not found."
exit 1
fi

resource_types=("Gateway" "Route" "Port_Binding")

for resource_type in "${resource_types[@]}"; do
uuid_array=($(ovn-ic-sbctl --columns=_uuid find $resource_type availability_zone=$availability_zone_uuid | awk '{print $3}'))

for uuid in "${uuid_array[@]}"; do
ovn-ic-sbctl destroy $resource_type $uuid
echo "Destroyed $resource_type: $uuid"
done
done

ovn-ic-sbctl destroy availability_zone $availability_zone_uuid
echo "Destroyed availability_zone: $availability_zone_uuid"

2 changes: 2 additions & 0 deletions dist/images/cleanup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ kubectl delete --ignore-not-found cm ovn-config ovn-ic-config ovn-external-gw-co
kubectl delete --ignore-not-found svc kube-ovn-pinger kube-ovn-controller kube-ovn-cni kube-ovn-monitor -n kube-system
kubectl delete --ignore-not-found ds kube-ovn-cni -n kube-system
kubectl delete --ignore-not-found deploy kube-ovn-controller -n kube-system
kubectl delete --ignore-not-found deploy ovn-ic-controller -n kube-system
kubectl delete --ignore-not-found deploy ovn-ic-server -n kube-system

# ensure kube-ovn-cni has been deleted
while :; do
Expand Down
148 changes: 148 additions & 0 deletions dist/images/install-ic-server.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
#!/usr/bin/env bash
set -euo pipefail

REGISTRY="kubeovn"
VERSION="v1.11.14"
TS_NUM=${TS_NUM:-3}
IMAGE_PULL_POLICY="IfNotPresent"
addresses=$(kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{print $6}' | tr \\n ',')
count=$(kubectl get no -lkube-ovn/role=master --no-headers | wc -l)
OVN_LEADER_PROBE_INTERVAL=${OVN_LEADER_PROBE_INTERVAL:-5}

cat <<EOF > ovn-ic-server.yaml
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: ovn-ic-server
namespace: kube-system
annotations:
kubernetes.io/description: |
OVN IC Server
spec:
replicas: $count
strategy:
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
type: RollingUpdate
selector:
matchLabels:
app: ovn-ic-server
template:
metadata:
labels:
app: ovn-ic-server
component: network
type: infra
spec:
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
- key: CriticalAddonsOnly
operator: Exists
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app: ovn-ic-server
topologyKey: kubernetes.io/hostname
priorityClassName: system-cluster-critical
serviceAccountName: ovn
hostNetwork: true
containers:
- name: ovn-ic-server
image: "$REGISTRY/kube-ovn:$VERSION"
imagePullPolicy: $IMAGE_PULL_POLICY
command: ["/kube-ovn/start-ic-db.sh"]
securityContext:
capabilities:
add: ["SYS_NICE"]
env:
- name: ENABLE_SSL
value: "false"
- name: TS_NUM
value: "$TS_NUM"
- name: NODE_IPS
value: $addresses
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: OVN_LEADER_PROBE_INTERVAL
value: "$OVN_LEADER_PROBE_INTERVAL"
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
resources:
requests:
cpu: 300m
memory: 200Mi
limits:
cpu: 3
memory: 1Gi
volumeMounts:
- mountPath: /var/run/ovn
name: host-run-ovn
- mountPath: /etc/ovn
name: host-config-ovn
- mountPath: /var/log/ovn
name: host-log-ovn
- mountPath: /etc/localtime
name: localtime
- mountPath: /var/run/tls
name: kube-ovn-tls
readinessProbe:
exec:
command:
- bash
- /kube-ovn/ovn-ic-healthcheck.sh
periodSeconds: 15
timeoutSeconds: 45
livenessProbe:
exec:
command:
- bash
- /kube-ovn/ovn-ic-healthcheck.sh
initialDelaySeconds: 30
periodSeconds: 15
failureThreshold: 5
timeoutSeconds: 4
nodeSelector:
kubernetes.io/os: "linux"
kube-ovn/role: "master"
volumes:
- name: host-run-ovn
hostPath:
path: /run/ovn
- name: host-config-ovn
hostPath:
path: /etc/origin/ovn
- name: host-log-ovn
hostPath:
path: /var/log/ovn
- name: localtime
hostPath:
path: /etc/localtime
- name: kube-ovn-tls
secret:
optional: true
secretName: kube-ovn-tls
EOF

kubectl apply -f ovn-ic-server.yaml
kubectl rollout status deployment/ovn-ic-server -n kube-system --timeout 600s

echo "OVN IC Server installed Successfully"

0 comments on commit 7b863fb

Please sign in to comment.