Skip to content

Commit

Permalink
underlay: fix link name exchange (#2516)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Mar 22, 2023
1 parent f22535e commit 8d0d56e
Show file tree
Hide file tree
Showing 6 changed files with 165 additions and 269 deletions.
215 changes: 28 additions & 187 deletions dist/images/start-ovs.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,194 +72,37 @@ ovsdb_server_ctl="/var/run/openvswitch/ovsdb-server.$(cat /var/run/openvswitch/o
ovs-appctl -t "$ovsdb_server_ctl" vlog/set jsonrpc:file:err
ovs-appctl -t "$ovsdb_server_ctl" vlog/set reconnect:file:err

function exchange_link_names() {
mappings=($(ovs-vsctl --if-exists get open . external-ids:ovn-bridge-mappings | tr -d '"' | tr ',' ' '))
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true))
function handle_underlay_bridges() {
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn))
for br in ${bridges[@]}; do
provider=""
for m in ${mappings[*]}; do
if echo $m | grep -q ":$br"'$'; then
provider=${m%:$br}
break
fi
done
if [ "x$provider" = "x" ]; then
echo "error: failed to get provider name for bridge $br"
continue
fi

port="br-$provider"
if ip link show $port 2>/dev/null; then
echo "link $port already exists"
continue
fi
if ! ip link show $br 2>/dev/null; then
echo "link $br does not exists"
continue
if ! ip link show $br >/dev/null; then
# the bridge does not exist, leave it to be handled by kube-ovn-cni
echo "deleting ovs bridge $br"
ovs-vsctl --no-wait del-br $br
fi

echo "change link name from $br to $port"
ipv4_routes=($(ip -4 route show dev $br | tr ' ' '#'))
ipv6_routes=($(ip -6 route show dev $br | tr ' ' '#'))
ip link set $br down
ip link set $br name $port
ip link set $port up

# transfer IPv4 routes
default_ipv4_routes=()
for route in ${ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv4 route $r to $port\n"
ip -4 route replace $r dev $port
else
default_ipv4_routes=(${default_ipv4_routes[@]} $route)
fi
done
for route in ${default_ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv4 route $r to $port\n"
ip -4 route replace $r dev $port
done

# transfer IPv6 routes
default_ipv6_routes=()
for route in ${ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv6 route $r to $port\n"
ip -6 route replace $r dev $port
else
default_ipv6_routes=(${default_ipv6_routes[@]} $route)
fi
done
for route in ${default_ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv6 route $r to $port\n"
ip -6 route replace $r dev $port
done
done
}

exchange_link_names

function wait_flows_pre_check() {
local devices=""
local ips=($(echo $OVN_DB_IPS | sed 's/,/ /g'))
for ip in ${ips[*]}; do
devices="$devices $(ip route get $ip | grep -oE 'dev .+' | awk '{print $2}')"
done

bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn))
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true))
for br in ${bridges[@]}; do
ports=($(ovs-vsctl list-ports $br))
for port in ${ports[@]}; do
if ! echo $devices | grep -qw "$port"; then
continue
fi

port_type=$(ovs-vsctl --no-heading --columns=type find interface name=$port)
if [ ! "x$port_type" = 'x""' ]; then
continue
fi

if ! ip link show $port | grep -qw "master ovs-system"; then
return 1
fi
done
if [ -z $(ip link show $br type openvswitch) ]; then
# the bridge does not exist, leave it to be handled by kube-ovn-cni
echo "deleting ovs bridge $br"
ovs-vsctl --no-wait del-br $br
fi
done

return 0
}

skip_wait_flows=0
if ! wait_flows_pre_check; then
skip_wait_flows=1
fi
handle_underlay_bridges

if [ $skip_wait_flows -eq 0 ]; then
# When ovs-vswitchd starts with this value set as true, it will neither flush or
# expire previously set datapath flows nor will it send and receive any
# packets to or from the datapath. Please check ovs-vswitchd.conf.db.5.txt
ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="true"
else
ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="false"
fi
# When ovs-vswitchd starts with this value set as true, it will neither flush or
# expire previously set datapath flows nor will it send and receive any
# packets to or from the datapath. Please check ovs-vswitchd.conf.db.5.txt
ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="true"

# Start vswitchd. restart will automatically set/unset flow-restore-wait which is not what we want
/usr/share/openvswitch/scripts/ovs-ctl start --no-ovsdb-server --system-id=random --no-mlockall
/usr/share/openvswitch/scripts/ovs-ctl --protocol=udp --dport=6081 enable-protocol

sleep 1

function handle_underlay_bridges() {
bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn))
for br in ${bridges[@]}; do
echo "handle bridge $br"
ip link set $br up

ports=($(ovs-vsctl list-ports $br))
for port in ${ports[@]}; do
port_type=$(ovs-vsctl --no-heading --columns=type find interface name=$port)
if [ ! "x$port_type" = 'x""' ]; then
continue
fi

echo "handle port $port on bridge $br"
ipv4_routes=($(ip -4 route show dev $port | tr ' ' '#'))
ipv6_routes=($(ip -6 route show dev $port | tr ' ' '#'))

set +o pipefail
addresses=($(ip addr show dev $port | grep -E '^\s*inet[6]?\s+' | grep -w global | awk '{print $2}'))
set -o pipefail

# transfer IP addresses
for addr in ${addresses[@]}; do
printf "delete address $addr on $port\n"
ip addr del $addr dev $port || true
printf "add/replace address $addr to $br\n"
ip addr replace $addr dev $br
done

# transfer IPv4 routes
default_ipv4_routes=()
for route in ${ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv4 route $r to $br\n"
ip -4 route replace $r dev $br
else
default_ipv4_routes=(${default_ipv4_routes[@]} $route)
fi
done
for route in ${default_ipv4_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv4 route $r to $br\n"
ip -4 route replace $r dev $br
done

# transfer IPv6 routes
default_ipv6_routes=()
for route in ${ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
if echo $r | grep -q -w 'scope link'; then
printf "add/replace IPv6 route $r to $br\n"
ip -6 route replace $r dev $br
else
default_ipv6_routes=(${default_ipv6_routes[@]} $route)
fi
done
for route in ${default_ipv6_routes[@]}; do
r=$(echo $route | tr '#' ' ')
printf "add/replace IPv6 route $r to $br\n"
ip -6 route replace $r dev $br
done
done
done
}

handle_underlay_bridges

function gen_conn_str {
if [[ -z "${OVN_DB_IPS}" ]]; then
if [[ "$ENABLE_SSL" == "false" ]]; then
Expand Down Expand Up @@ -291,21 +134,19 @@ else
/usr/share/ovn/scripts/ovn-ctl --ovn-controller-ssl-key=/var/run/tls/key --ovn-controller-ssl-cert=/var/run/tls/cert --ovn-controller-ssl-ca-cert=/var/run/tls/cacert restart_controller
fi

if [ $skip_wait_flows -eq 0 ]; then
# Wait ovn-controller finish init flow compute and update it to vswitchd,
# then update flow-restore-wait to indicate vswitchd to process flows
set +e
# Wait ovn-controller finish init flow compute and update it to vswitchd,
# then update flow-restore-wait to indicate vswitchd to process flows
set +e
flow_num=$(ovs-ofctl dump-flows br-int | wc -l)
while [ $flow_num -le $FLOW_LIMIT ]
do
echo "$flow_num flows now, waiting for ovs-vswitchd flow ready"
sleep 1
flow_num=$(ovs-ofctl dump-flows br-int | wc -l)
while [ $flow_num -le $FLOW_LIMIT ]
do
echo "$flow_num flows now, waiting for ovs-vswitchd flow ready"
sleep 1
flow_num=$(ovs-ofctl dump-flows br-int | wc -l)
done
set -e
done
set -e

ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="false"
fi
ovs-vsctl --no-wait set open_vswitch . other_config:flow-restore-wait="false"

set +e
for netns in /var/run/netns/*; do
Expand Down
4 changes: 4 additions & 0 deletions pkg/daemon/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,10 @@ func (c *Controller) recordProviderNetworkErr(providerNetwork string, errMsg str
break
}
}
if currentPod == nil {
klog.Warning("failed to get self pod")
return
}
} else {
if currentPod, err = c.config.KubeClient.CoreV1().Pods(c.localNamespace).Get(context.Background(), c.localPodName, metav1.GetOptions{}); err != nil {
klog.Errorf("failed to get pod %s, %v", c.localPodName, err)
Expand Down
37 changes: 13 additions & 24 deletions pkg/daemon/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,34 +147,11 @@ func ovsCleanProviderNetwork(provider string) error {
for idx, m = range brMappings {
if strings.HasPrefix(m, mappingPrefix) {
brName = m[len(mappingPrefix):]
klog.V(3).Infof("found bridge name for provider %s: %s", provider, brName)
break
}
}

if output, err = ovs.Exec("list-br"); err != nil {
return fmt.Errorf("failed to list OVS bridge %v: %q", err, output)
}

if !util.ContainsString(strings.Split(output, "\n"), brName) {
return nil
}

// get host nic
if output, err = ovs.Exec("list-ports", brName); err != nil {
return fmt.Errorf("failed to list ports of OVS bridge %s, %v: %q", brName, err, output)
}

// remove host nic from the external bridge
if output != "" {
for _, port := range strings.Split(output, "\n") {
if err = removeProviderNic(port, brName); err != nil {
errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err)
klog.Error(errMsg)
return errMsg
}
}
}

if idx != len(brMappings) {
brMappings = append(brMappings[:idx], brMappings[idx+1:]...)
if len(brMappings) == 0 {
Expand Down Expand Up @@ -206,6 +183,15 @@ func ovsCleanProviderNetwork(provider string) error {
return fmt.Errorf("failed to set ovn-chassis-mac-mappings, %v: %q", err, output)
}

if output, err = ovs.Exec("list-br"); err != nil {
return fmt.Errorf("failed to list OVS bridge %v: %q", err, output)
}

if !util.ContainsString(strings.Split(output, "\n"), brName) {
klog.V(3).Infof("ovs bridge %s not found", brName)
return nil
}

// get host nic
if output, err = ovs.Exec("list-ports", brName); err != nil {
return fmt.Errorf("failed to list ports of OVS bridge %s, %v: %q", brName, err, output)
Expand All @@ -214,18 +200,21 @@ func ovsCleanProviderNetwork(provider string) error {
// remove host nic from the external bridge
if output != "" {
for _, port := range strings.Split(output, "\n") {
klog.V(3).Infof("removing ovs port %s from bridge %s", port, brName)
if err = removeProviderNic(port, brName); err != nil {
errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err)
klog.Error(errMsg)
return errMsg
}
klog.V(3).Infof("ovs port %s has been removed from bridge %s", port, brName)
}
}

// remove OVS bridge
if output, err = ovs.Exec(ovs.IfExists, "del-br", brName); err != nil {
return fmt.Errorf("failed to remove OVS bridge %s, %v: %q", brName, err, output)
}
klog.V(3).Infof("ovs bridge %s has been deleted", brName)

if br := util.ExternalBridgeName(provider); br != brName {
if _, err = changeProvideNicName(br, brName); err != nil {
Expand Down
Loading

0 comments on commit 8d0d56e

Please sign in to comment.