Skip to content

Commit

Permalink
northd: fix race condition in health check (#2154)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Dec 12, 2022
1 parent 755a46a commit 5444126
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 146 deletions.
167 changes: 25 additions & 142 deletions .github/workflows/build-x86-image.yaml
View file
Expand Up @@ -166,13 +166,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install
run: sudo make kind-install

- name: Run E2E
run: |
Expand Down Expand Up @@ -319,13 +313,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-iptables
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install
run: sudo make kind-install

- name: Run E2E
run: |
Expand Down Expand Up @@ -390,13 +378,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install
run: sudo make kind-install

- name: Run E2E
run: |
Expand Down Expand Up @@ -461,13 +443,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install-underlay
run: sudo make kind-install-underlay

- name: Run E2E
run: |
Expand Down Expand Up @@ -555,13 +531,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-single
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install
run: sudo make kind-install

- name: Run E2E
run: |
Expand Down Expand Up @@ -626,17 +596,12 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-ha
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install
sudo kubectl patch subnet ovn-default --type merge \
-p '{"spec":{"gatewayType": "centralized", "gatewayNode": "kube-ovn-control-plane"}}'
sudo kubectl -n kube-system patch svc kube-ovn-pinger --type merge \
-p '{"spec":{"type": "NodePort", "externalTrafficPolicy": "Local"}}'
run: |
sudo make kind-install
sudo kubectl patch subnet ovn-default --type merge \
-p '{"spec":{"gatewayType": "centralized", "gatewayNode": "kube-ovn-control-plane"}}'
sudo kubectl -n kube-system patch svc kube-ovn-pinger --type merge \
-p '{"spec":{"type": "NodePort", "externalTrafficPolicy": "Local"}}'
- name: Run E2E
run: |
Expand Down Expand Up @@ -708,13 +673,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-ipv6
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install-ipv6
run: sudo make kind-install-ipv6

- name: Run E2E
run: |
Expand Down Expand Up @@ -786,13 +745,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-ipv6
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install-ipv6
run: sudo make kind-install-ipv6

- name: Run E2E
run: |
Expand Down Expand Up @@ -857,13 +810,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-ipv6
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install-underlay-ipv6
run: sudo make kind-install-underlay-ipv6

- name: Run E2E
run: |
Expand Down Expand Up @@ -951,13 +898,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-dual
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install-dual
run: sudo make kind-install-dual

- name: Run E2E
run: |
Expand Down Expand Up @@ -1029,13 +970,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-dual
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install-underlay-dual
run: sudo make kind-install-underlay-dual

- name: Run E2E
run: |
Expand Down Expand Up @@ -1123,13 +1058,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-dual
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install-underlay-hairpin-dual
run: sudo make kind-install-underlay-hairpin-dual

- name: Run E2E
run: |
Expand Down Expand Up @@ -1189,13 +1118,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-dual
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install-underlay-logical-gateway-dual
run: sudo make kind-install-underlay-logical-gateway-dual

- name: Run E2E
run: |
Expand Down Expand Up @@ -1283,13 +1206,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init
- name: Install Kube-OVN without LoadBalancer
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo ENABLE_LB=false make kind-install
run: sudo ENABLE_LB=false make kind-install

- name: Run E2E
run: |
Expand Down Expand Up @@ -1354,13 +1271,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-iptables
- name: Install Kube-OVN without LoadBalancer
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo ENABLE_LB=false make kind-install
run: sudo ENABLE_LB=false make kind-install

- name: Run E2E
run: |
Expand Down Expand Up @@ -1425,13 +1336,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo ENABLE_NP=false make kind-install
run: sudo ENABLE_NP=false make kind-install

- name: Run E2E
run: |
Expand Down Expand Up @@ -1507,13 +1412,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init
- name: Install Kube-OVN & Multus
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo make kind-install-multus
run: sudo make kind-install-multus

- name: Run E2E
run: |
Expand Down Expand Up @@ -1573,13 +1472,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-ovn-ic
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: |
sudo PATH=~/.local/bin:$PATH make kind-install-ovn-ic
run: sudo PATH=~/.local/bin:$PATH make kind-install-ovn-ic

- name: Run E2E
run: |
Expand Down Expand Up @@ -1647,12 +1540,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH make kind-init-cilium
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: sudo PATH=~/.local/bin:$PATH make kind-install-cilium
run: sudo PATH=~/.local/bin:$PATH make kind-install-cilium

- name: Run E2E
run: |
Expand Down Expand Up @@ -1699,12 +1587,7 @@ jobs:
sudo PATH=~/.local/bin:$PATH k8s_version=v1.23.13 make kind-init
- name: Install Kube-OVN
uses: nick-invision/retry@v2
with:
timeout_minutes: 10
max_attempts: 3
shell: bash
command: sudo PATH=~/.local/bin:$PATH make kind-install
run: sudo PATH=~/.local/bin:$PATH make kind-install

- name: Cleanup
run: |
Expand Down
9 changes: 5 additions & 4 deletions pkg/ovn_leader_checker/ovn.go
View file
Expand Up @@ -382,14 +382,15 @@ func doOvnLeaderCheck(cfg *Configuration, podName string, podNamespace string) {
for k, v := range cachedPod.Labels {
labels[k] = v
}
updatePodLabels(labels, "ovn-nb-leader", checkNbIsLeader())
updatePodLabels(labels, "ovn-sb-leader", checkSbIsLeader())
updatePodLabels(labels, "ovn-northd-leader", checkNorthdActive())
nbLeader, sbLeader, northdLeader := checkNbIsLeader(), checkSbIsLeader(), checkNorthdActive()
updatePodLabels(labels, "ovn-nb-leader", nbLeader)
updatePodLabels(labels, "ovn-sb-leader", sbLeader)
updatePodLabels(labels, "ovn-northd-leader", northdLeader)
if err = patchPodLabels(cfg, cachedPod, labels); err != nil {
klog.Errorf("patch label error %v", err)
return
}
if checkNorthdSvcExist(cfg, podNamespace, "ovn-northd") {
if sbLeader && checkNorthdSvcExist(cfg, podNamespace, "ovn-northd") {
if !checkNorthdSvcValidIP(cfg, podNamespace, "ovn-northd") {
klog.Warning("no available northd leader, try to release the lock")
stealLock()
Expand Down

0 comments on commit 5444126

Please sign in to comment.