ovn: add nb option version_compatibility (#3671)

Signed-off-by: zhangzujian <zhangzujian.7@gmail.com>

charts/templates/central-deploy.yaml

@@ -80,6 +80,10 @@ spec:
               value: "{{ .Values.networking.OVN_NORTHD_N_THREADS }}"
             - name: ENABLE_COMPACT
               value: "{{ .Values.networking.ENABLE_COMPACT }}"
+            {{- if include "kubeovn.ovs-ovn.updateStrategy" . | eq "OnDelete" }}
+            - name: OVN_VERSION_COMPATIBILITY
+              value: "22.03"
+            {{- end }}
           resources:
             requests:
             {{- if .Release.IsUpgrade }}

charts/templates/upgrade-ovs-ovn.yaml

@@ -32,6 +32,16 @@ rules:
       - ovs-ovn
     verbs:
       - get
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+    resourceNames:
+      - ovn-central
+    verbs:
+      - get
+      - list
+      - watch
   - apiGroups:
       - ""
     resources:
@@ -127,6 +137,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: ENABLE_SSL
+              value: "{{ .Values.networking.ENABLE_SSL }}"
+            - name: OVN_DB_IPS
+              value: "{{ .Values.MASTER_NODES | default (include "kubeovn.nodeIPs" .) }}"
           command:
             - bash
             - -eo
@@ -136,8 +150,14 @@ spec:
           volumeMounts:
             - mountPath: /var/log/kube-ovn
               name: kube-ovn-log
+            - mountPath: /var/run/tls
+              name: kube-ovn-tls
       volumes:
         - name: kube-ovn-log
           hostPath:
             path: {{ .Values.log_conf.LOG_DIR }}/kube-ovn
+        - name: kube-ovn-tls
+          secret:
+            optional: true
+            secretName: kube-ovn-tls
 {{ end }}

dist/images/Dockerfile.base

@@ -46,7 +46,9 @@ RUN cd /usr/src/ && git clone -b branch-22.12 --depth=1 https://github.com/ovn-o
     # ovn-ic blacklist function not work on ipv6
     curl -s https://github.com/kubeovn/ovn/commit/78ab91005854532e7eb5c4fe6b2923ce292e3681.patch | git apply && \
     # lflow: do not send direct traffic between lports to conntrack
-    curl -s https://github.com/kubeovn/ovn/commit/6f1af045845deeabf06fdc7c90073e0a6874ab2f.patch | git apply
+    curl -s https://github.com/kubeovn/ovn/commit/6f1af045845deeabf06fdc7c90073e0a6874ab2f.patch | git apply && \
+    # northd: add nb option version_compatibility
+    curl -s https://github.com/kubeovn/ovn/commit/174561abd707239f134501c4320cbf9b29af0305.patch | git apply
 
 RUN apt install -y build-essential fakeroot \
     autoconf automake bzip2 debhelper-compat dh-exec dh-python dh-sequence-python3 dh-sequence-sphinxdoc \
@@ -80,7 +82,7 @@ FROM ubuntu:22.04
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname libunwind8 netbase \
         ethtool iproute2 ncat libunbound-dev procps libatomic1 kmod iptables python3-netifaces python3-sortedcontainers \
-        tcpdump ipset curl uuid-runtime openssl inetutils-ping arping ndisc6 \
+        tcpdump ipset curl uuid-runtime openssl inetutils-ping arping ndisc6 conntrack \
         logrotate dnsutils net-tools strongswan strongswan-pki libcharon-extra-plugins libmnl-dev \
         libcharon-extauth-plugins libstrongswan-extra-plugins libstrongswan-standard-plugins -y --no-install-recommends && \
         rm -rf /var/lib/apt/lists/* && \

dist/images/start-db.sh

@@ -2,7 +2,10 @@
 set -eo pipefail
 
 DEBUG_WRAPPER=${DEBUG_WRAPPER:-}
+ENABLE_COMPACT=${ENABLE_COMPACT:-false}
+PROBE_INTERVAL=${PROBE_INTERVAL:-180000}
 OVN_NORTHD_N_THREADS=${OVN_NORTHD_N_THREADS:-1}
+OVN_VERSION_COMPATIBILITY=${OVN_VERSION_COMPATIBILITY:-}
 DEBUG_OPT="--ovn-northd-wrapper=$DEBUG_WRAPPER --ovsdb-nb-wrapper=$DEBUG_WRAPPER --ovsdb-sb-wrapper=$DEBUG_WRAPPER"
 
 echo "PROBE_INTERVAL is set to $PROBE_INTERVAL"
@@ -44,6 +47,13 @@ if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then
     DB_ADDRESSES="$POD_IPS"
 fi
 
+SSL_OPTIONS=
+function ssl_options() {
+    if "$ENABLE_SSL" != "false" ]; then
+        SSL_OPTIONS="-p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert"
+    fi
+}
+
 . /usr/share/openvswitch/scripts/ovs-lib || exit 1
 
 function random_str {
@@ -106,7 +116,7 @@ function ovndb_query_leader {
     if [[ "$ENABLE_SSL" == "false" ]]; then 
         timeout 10 ovsdb-client query $(gen_conn_addr $i $port) "$query"
     else
-        timeout 10 ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query $(gen_conn_addr $i $port) "$query"
+        timeout 10 ovsdb-client $SSL_OPTIONS query $(gen_conn_addr $i $port) "$query"
     fi
 }
 
@@ -125,6 +135,14 @@ function is_clustered {
   return 1
 }
 
+function set_nb_version_compatibility() {
+    if [ -n "$OVN_VERSION_COMPATIBILITY" ]; then
+        if ! ovn-nbctl --db=$(gen_conn_str 6641) $SSL_OPTIONS get NB_Global . options | grep -qw version_compatibility=; then
+            ovn-nbctl --db=$(gen_conn_str 6641) $SSL_OPTIONS set NB_Global . options:version_compatibility=${OVN_VERSION_COMPATIBILITY}
+        fi
+    fi
+}
+
 # create a new db file and join it to the cluster
 # if the nb/sb db file is corrputed
 function ovn_db_pre_start() {
@@ -237,7 +255,7 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
         ovn-sbctl --no-leader-only set-connection ptcp:"${SB_PORT}":["${DB_ADDR}"]
         ovn-sbctl --no-leader-only set Connection . inactivity_probe=${PROBE_INTERVAL}
     else
-        if [[ ! "$NODE_IPS" =~ "$DB_CLUSTER_ADDR" ]]; then
+        if ! echo "$NODE_IPS" | tr ',' '\n' | grep '^'`echo "$DB_CLUSTER_ADDR" | sed 's/\./\\\./g'`'$'; then
             echo "ERROR! host ip $DB_CLUSTER_ADDR not in env NODE_IPS $NODE_IPS"
             exit 1
         fi
@@ -251,7 +269,7 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
         is_clustered
         result=$?
         set -eo pipefail
-        # leader up only when no cluster and on first node
+        # leader up only when no cluster and on the first/only node
         if [[ ${result} -eq 1 && "$nb_leader_ip" == "$DB_CLUSTER_ADDR" ]]; then
             ovn_ctl_args="$DEBUG_OPT \
                 --db-nb-create-insecure-remote=yes \
@@ -277,6 +295,7 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
                 start_sb_ovsdb -- \
                 --remote=db:Local_Config,Config,connections \
                 /etc/ovn/ovnsb_local_config.db
+            set_nb_version_compatibility
             /usr/share/ovn/scripts/ovn-ctl $ovn_ctl_args \
                 --ovn-manage-ovsdb=no --ovn-northd-n-threads="${OVN_NORTHD_N_THREADS}" start_northd
             ovn-nbctl --no-leader-only set NB_Global . options:inactivity_probe=${PROBE_INTERVAL}
@@ -341,6 +360,7 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
                 -- \
                 --remote=db:Local_Config,Config,connections \
                 /etc/ovn/ovnsb_local_config.db
+            set_nb_version_compatibility
             /usr/share/ovn/scripts/ovn-ctl \
                 $ovn_ctl_args \
                 --ovn-manage-ovsdb=no \
@@ -362,14 +382,14 @@ else
             --ovn-northd-ssl-ca-cert=/var/run/tls/cacert \
             --ovn-northd-n-threads="${OVN_NORTHD_N_THREADS}" \
             restart_northd
-        ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${NB_PORT}":["${DB_ADDR}"]
-        ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=${PROBE_INTERVAL}
-        ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set NB_Global . options:use_logical_dp_groups=true
+        ovn-nbctl --no-leader-only $SSL_OPTIONS set-connection pssl:"${NB_PORT}":["${DB_ADDR}"]
+        ovn-nbctl --no-leader-only $SSL_OPTIONS set Connection . inactivity_probe=${PROBE_INTERVAL}
+        ovn-nbctl --no-leader-only $SSL_OPTIONS set NB_Global . options:use_logical_dp_groups=true
 
-        ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${SB_PORT}":["${DB_ADDR}"]
-        ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=${PROBE_INTERVAL}
+        ovn-sbctl --no-leader-only $SSL_OPTIONS set-connection pssl:"${SB_PORT}":["${DB_ADDR}"]
+        ovn-sbctl --no-leader-only $SSL_OPTIONS set Connection . inactivity_probe=${PROBE_INTERVAL}
     else
-        if [[ ! "$NODE_IPS" =~ "$DB_CLUSTER_ADDR" ]]; then
+        if ! echo "$NODE_IPS" | tr ',' '\n' | grep '^'`echo "$DB_CLUSTER_ADDR" | sed 's/\./\\\./g'`'$'; then
             echo "ERROR! host ip $DB_CLUSTER_ADDR not in env NODE_IPS $NODE_IPS"
             exit 1
         fi
@@ -419,8 +439,8 @@ else
                 /etc/ovn/ovnsb_local_config.db
             /usr/share/ovn/scripts/ovn-ctl $ovn_ctl_args \
                 --ovn-manage-ovsdb=no --ovn-northd-n-threads="${OVN_NORTHD_N_THREADS}" start_northd
-            ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set NB_Global . options:northd_probe_interval=${PROBE_INTERVAL}
-            ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set NB_Global . options:use_logical_dp_groups=true
+            ovn-nbctl --no-leader-only $SSL_OPTIONS set NB_Global . options:northd_probe_interval=${PROBE_INTERVAL}
+            ovn-nbctl --no-leader-only $SSL_OPTIONS set NB_Global . options:use_logical_dp_groups=true
         else
             # get leader if cluster exists
             set +eo pipefail
@@ -485,6 +505,7 @@ else
                 start_sb_ovsdb -- \
                 --remote=db:Local_Config,Config,connections \
                 /etc/ovn/ovnsb_local_config.db
+            set_nb_version_compatibility
             /usr/share/ovn/scripts/ovn-ctl $ovn_ctl_args \
                 --ovn-manage-ovsdb=no --ovn-northd-n-threads="${OVN_NORTHD_N_THREADS}" start_northd
         fi

dist/images/upgrade-ovs.sh

@@ -2,8 +2,47 @@
 
 set -ex
 
+OVN_DB_IPS=${OVN_DB_IPS:-}
+ENABLE_SSL=${ENABLE_SSL:-false}
 POD_NAMESPACE=${POD_NAMESPACE:-kube-system}
 
+SSL_OPTIONS=
+function ssl_options() {
+    if "$ENABLE_SSL" != "false" ]; then
+        SSL_OPTIONS="-p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert"
+    fi
+}
+
+function gen_conn_str {
+  if [[ -z "${OVN_DB_IPS}" ]]; then
+    if [[ "$ENABLE_SSL" == "false" ]]; then
+      x="tcp:[${OVN_NB_SERVICE_HOST}]:${OVN_NB_SERVICE_PORT}"
+    else
+      x="ssl:[${OVN_NB_SERVICE_HOST}]:${OVN_NB_SERVICE_PORT}"
+    fi
+  else
+    t=$(echo -n "${OVN_DB_IPS}" | sed 's/[[:space:]]//g' | sed 's/,/ /g')
+    if [[ "$ENABLE_SSL" == "false" ]]; then
+      x=$(for i in ${t}; do echo -n "tcp:[$i]:$1,"; done | sed 's/,$//')
+    else
+      x=$(for i in ${t}; do echo -n "ssl:[$i]:$1,"; done | sed 's/,$//')
+    fi
+  fi
+  echo "$x"
+}
+
+nb_addr="$(gen_conn_str 6641)"
+while true; do
+  if [ x`ovn-nbctl --db=$nb_addr $SSL_OPTIONS get nb . options | grep -o 'version_compatibility='` != "x" ]; then
+    echo "ovn nb global option version_compatibility is set to "`ovn-nbctl --db=$nb_addr $SSL_OPTIONS get nb . options:version_compatibility`
+    break
+  fi
+  echo "waiting for ovn nb global option version_compatibility to be set..."
+  sleep 3
+done
+
+kubectl -n $POD_NAMESPACE rollout status deploy ovn-central --timeout=60s
+
 dsChartVer=`kubectl get ds -n $POD_NAMESPACE ovs-ovn -o jsonpath={.spec.template.metadata.annotations.chart-version}`
 
 for node in `kubectl get node -o jsonpath='{.items[*].metadata.name}'`; do
@@ -28,3 +67,5 @@ for node in `kubectl get node -o jsonpath='{.items[*].metadata.name}'`; do
   echo "waiting for ovs-ovn pod on node $node to be ready"
   kubectl -n $POD_NAMESPACE wait pod --for=condition=ready -l app=ovs --field-selector spec.nodeName=$node
 done
+
+ovn-nbctl --db=$nb_addr $SSL_OPTIONS set nb . options:version_compatibility=none