ovn db: add support for listening on pod ip (#2235)

.github/workflows/build-x86-image.yaml

@@ -584,8 +584,8 @@ jobs:
         run: sh dist/images/cleanup.sh
 
   underlay-logical-gateway-installation-test:
-    needs: build-kube-ovn
     name: Underlay Logical Gateway Installation Test
+    needs: build-kube-ovn
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     steps:
@@ -620,8 +620,8 @@ jobs:
         run: sh dist/images/cleanup.sh
 
   no-ovn-lb-test:
-    needs: build-kube-ovn
     name: Disable OVN LB Test
+    needs: build-kube-ovn
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     steps:
@@ -658,8 +658,8 @@ jobs:
         run: sh dist/images/cleanup.sh
 
   no-np-test:
-    needs: build-kube-ovn
     name: Disable Network Policy Test
+    needs: build-kube-ovn
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     steps:
@@ -696,8 +696,8 @@ jobs:
         run: sh dist/images/cleanup.sh
 
   installation-compatibility-test:
-    needs: build-kube-ovn
     name: Installation Compatibility Test
+    needs: build-kube-ovn
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     steps:
@@ -732,8 +732,8 @@ jobs:
         run: sh dist/images/cleanup.sh
 
   cilium-chaining-e2e:
-    needs: build-kube-ovn
     name: Cilium Chaining E2E
+    needs: build-kube-ovn
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     steps:
@@ -819,7 +819,106 @@ jobs:
       - name: Cleanup
         run: sh dist/images/cleanup.sh
 
+  kube-ovn-security-e2e:
+    name: Kube-OVN Security E2E
+    needs: build-kube-ovn
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        ssl:
+          - "true"
+          - "false"
+        bind-local:
+          - "true"
+          - "false"
+        ip-family:
+          - ipv4
+          - ipv6
+          - dual
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create the default branch directory
+        run: mkdir -p test/e2e/source
+
+      - name: Check out the default branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          fetch-depth: 1
+          path: test/e2e/source
+
+      - name: Export E2E directory
+        run: |
+          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
+            echo "E2E_DIR=." >> "$GITHUB_ENV"
+          else
+            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
+          fi
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '${{ env.GO_VERSION }}'
+          check-latest: true
+        id: go
+
+      - name: Export Go full version
+        run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"
+
+      - name: Go cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles('${{ env.E2E_DIR }}/**/go.sum') }}
+          restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-
+
+      - name: Build e2e binaries
+        working-directory: ${{ env.E2E_DIR }}
+        run: make e2e-compile
+
+      - name: Install kind
+        run: |
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+
+      - name: Download image
+        uses: actions/download-artifact@v3
+        with:
+          name: kube-ovn
+
+      - name: Load image
+        run: docker load --input kube-ovn.tar
+
+      - name: Create kind cluster
+        run: |
+          sudo pip3 install j2cli
+          sudo pip3 install "j2cli[yaml]"
+          sudo PATH=~/.local/bin:$PATH make kind-init-ha-${{ matrix.ip-family }}
+          sudo cp -r /root/.kube/ ~/.kube/
+          sudo chown -R $(id -un). ~/.kube/
+
+      - name: Install Kube-OVN
+        run: |
+          sudo ENABLE_SSL=${{ matrix.ssl }} ENABLE_BIND_LOCAL_IP=${{ matrix.bind-local }} \
+            make kind-install-${{ matrix.ip-family }}
+
+      - name: Run E2E
+        working-directory: ${{ env.E2E_DIR }}
+        env:
+          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
+          E2E_IP_FAMILY: ${{ matrix.ip-family }}
+        run: make kube-ovn-security-e2e
+
+      - name: Cleanup
+        run: sh dist/images/cleanup.sh
+
   push:
+    name: Push Images
     needs:
       - k8s-conformance-e2e
       # - k8s-netpol-e2e
@@ -833,7 +932,7 @@ jobs:
       - no-ovn-lb-test
       - no-np-test
       - cilium-chaining-e2e
-    name: push
+      - kube-ovn-security-e2e
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3

Makefile

@@ -209,9 +209,20 @@ kind-init-iptables:
 	@kube_proxy_mode=iptables $(MAKE) kind-init
 
 .PHONY: kind-init-ha
-kind-init-ha:
+kind-init-ha: kind-init-ha-ipv4
+
+.PHONY: kind-init-ha-ipv4
+kind-init-ha-ipv4:
 	@ha=true $(MAKE) kind-init
 
+.PHONY: kind-init-ha-ipv6
+kind-init-ha-ipv6:
+	@ip_family=ipv6 $(MAKE) kind-init-ha
+
+.PHONY: kind-init-ha-dual
+kind-init-ha-dual:
+	@ip_family=dual $(MAKE) kind-init-ha
+
 .PHONY: kind-init-single
 kind-init-single:
 	@single=true $(MAKE) kind-init

dist/images/Dockerfile.base

@@ -35,6 +35,8 @@ RUN dpkg -i /usr/src/python3-openvswitch*.deb /usr/src/libopenvswitch*.deb
 
 RUN cd /usr/src/ && git clone -b branch-21.06 --depth=1 https://github.com/ovn-org/ovn.git && \
     cd ovn && \
+    # fix ssl listen address
+    curl -s https://github.com/kubeovn/ovn/commit/62d4969877712c26fe425698d898b440f91b44bf.patch | git apply && \
     # expr.c: Use expr_destroy and expr_clone instead of free and xmemdup.
     curl -s https://github.com/ovn-org/ovn/commit/4b4cadcfabbc79f7d69c213be2b37e5e8634201c.patch | git apply && \
     # treewide: bump ovs and fix problematic loops

dist/images/install.sh

@@ -1158,6 +1158,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "$ENABLE_BIND_LOCAL_IP"
           resources:
             requests:
               cpu: 300m
@@ -1635,6 +1641,12 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
+            - name: ENABLE_BIND_LOCAL_IP
+              value: "$ENABLE_BIND_LOCAL_IP"
           resources:
             requests:
               cpu: 300m

dist/images/ovn-is-leader.sh

@@ -8,12 +8,14 @@ ovn-ctl status_northd
 ovn-ctl status_ovnnb
 ovn-ctl status_ovnsb
 
+BIND_LOCAL_ADDR=[${POD_IP:-127.0.0.1}]
+
 # For data consistency, only store leader address in endpoint
 # Store ovn-nb leader to svc kube-system/ovn-nb
 if [[ "$ENABLE_SSL" == "false" ]]; then
-  nb_leader=$(ovsdb-client query tcp:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
+  nb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 else
-  nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
+  nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 fi
 
 if [[ $nb_leader =~ "true" ]]
@@ -34,9 +36,9 @@ fi
 
 # Store ovn-sb leader to svc kube-system/ovn-sb
 if [[ "$ENABLE_SSL" == "false" ]]; then
-  sb_leader=$(ovsdb-client query tcp:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
+  sb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 else
-  sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
+  sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
 fi
 
 if [[ $sb_leader =~ "true" ]]
@@ -51,9 +53,9 @@ then
     if [ "$northd_leader" == "" ]; then
        # no available northd leader try to release the lock
        if [[ "$ENABLE_SSL" == "false" ]]; then
-         ovsdb-client -v -t 1 steal tcp:127.0.0.1:6642  ovn_northd
+         ovsdb-client -v -t 1 steal tcp:$BIND_LOCAL_ADDR:6642  ovn_northd
        else
-         ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:127.0.0.1:6642  ovn_northd
+         ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:$BIND_LOCAL_ADDR:6642  ovn_northd
        fi
      fi
    fi

dist/images/start-db.sh

@@ -25,6 +25,14 @@ DB_NB_PORT=${DB_NB_PORT:-6641}
 DB_SB_ADDR=${DB_SB_ADDR:-::}
 DB_SB_PORT=${DB_SB_PORT:-6642}
 ENABLE_SSL=${ENABLE_SSL:-false}
+ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-false}
+BIND_LOCAL_ADDR=[::]
+if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then
+  POD_IPS_LIST=(${POD_IPS//,/ })
+  if [[ ${#POD_IPS_LIST[@]} == 1 ]]; then
+    BIND_LOCAL_ADDR="[${POD_IP}]"
+  fi
+fi
 
 . /usr/share/openvswitch/scripts/ovs-lib || exit 1
 
@@ -177,8 +185,10 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
                 --db-sb-create-insecure-remote=yes \
                 --db-nb-cluster-local-addr="[${POD_IP}]" \
                 --db-sb-cluster-local-addr="[${POD_IP}]" \
-                --db-nb-addr=[::] \
-                --db-sb-addr=[::] \
+                --db-nb-addr=$BIND_LOCAL_ADDR \
+                --db-sb-addr=$BIND_LOCAL_ADDR \
+                --db-nb-use-remote-in-db=no \
+                --db-sb-use-remote-in-db=no \
                 --ovn-northd-nb-db="$(gen_conn_str 6641)" \
                 --ovn-northd-sb-db="$(gen_conn_str 6642)" \
                 start_northd
@@ -222,8 +232,10 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
                 --db-sb-cluster-local-addr="[${POD_IP}]" \
                 --db-nb-cluster-remote-addr="[${nb_leader_ip}]" \
                 --db-sb-cluster-remote-addr="[${sb_leader_ip}]" \
-                --db-nb-addr=[::] \
-                --db-sb-addr=[::] \
+                --db-nb-addr=$BIND_LOCAL_ADDR \
+                --db-sb-addr=$BIND_LOCAL_ADDR \
+                --db-nb-use-remote-in-db=no \
+                --db-sb-use-remote-in-db=no \
                 --ovn-northd-nb-db="$(gen_conn_str 6641)" \
                 --ovn-northd-sb-db="$(gen_conn_str 6642)" \
                 start_northd
@@ -277,16 +289,18 @@ else
                 --ovn-northd-ssl-ca-cert=/var/run/tls/cacert \
                 --db-nb-cluster-local-addr="[${POD_IP}]" \
                 --db-sb-cluster-local-addr="[${POD_IP}]" \
-                --db-nb-addr=[::] \
-                --db-sb-addr=[::] \
+                --db-nb-addr=$BIND_LOCAL_ADDR \
+                --db-sb-addr=$BIND_LOCAL_ADDR \
+                --db-nb-use-remote-in-db=no \
+                --db-sb-use-remote-in-db=no \
                 --ovn-northd-nb-db="$(gen_conn_str 6641)" \
                 --ovn-northd-sb-db="$(gen_conn_str 6642)" \
                 start_northd
-            ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":[::]
+            ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":["${DB_NB_ADDR}"]
             ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000
             ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set NB_Global . options:use_logical_dp_groups=true
 
-            ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":[::]
+            ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":["${DB_SB_ADDR}"]
             ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000
         else
             # get leader if cluster exists
@@ -328,8 +342,10 @@ else
                 --db-sb-cluster-local-addr="[${POD_IP}]" \
                 --db-nb-cluster-remote-addr="[${nb_leader_ip}]" \
                 --db-sb-cluster-remote-addr="[${sb_leader_ip}]" \
-                --db-nb-addr=[::] \
-                --db-sb-addr=[::] \
+                --db-nb-addr=$BIND_LOCAL_ADDR \
+                --db-sb-addr=$BIND_LOCAL_ADDR \
+                --db-nb-use-remote-in-db=no \
+                --db-sb-use-remote-in-db=no \
                 --ovn-northd-nb-db="$(gen_conn_str 6641)" \
                 --ovn-northd-sb-db="$(gen_conn_str 6642)" \
                 start_northd

yamls/ovn-dpdk.yaml

@@ -244,6 +244,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
           resources:
             requests:
               cpu: 500m

yamls/ovn-ha.yaml

@@ -263,6 +263,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
           resources:
             requests:
               cpu: 500m

yamls/ovn.yaml

@@ -283,6 +283,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+            - name: POD_IPS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIPs
           resources:
             requests:
               cpu: 500m