From 6b5345ff2aae3a6c6072593d444674bb0752909b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E7=A5=96=E5=BB=BA?= Date: Mon, 29 May 2023 09:36:05 +0800 Subject: [PATCH] ci: fix valgrind result analysis (#2853) --- .github/workflows/build-x86-image.yaml | 149 +++++++++++++++++++++++-- 1 file changed, 137 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml index 75aba18d085..6d941c670fc 100644 --- a/.github/workflows/build-x86-image.yaml +++ b/.github/workflows/build-x86-image.yaml @@ -389,19 +389,29 @@ jobs: kubectl -n kube-system rollout restart ds ovs-ovn kubectl -n kube-system rollout status deploy ovn-central kubectl -n kube-system rollout status ds ovs-ovn + while true; do + if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then + break + fi + sleep 1 + done kubectl ko log ovn kubectl ko log ovs - exit_code=0 - find kubectl-ko-log -type f -name '*.valgrind.*' | while read f; do + for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do + echo "Checking if valgrind log file for $daemon exists..." + find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 + done + + find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do if grep -qw 'definitely lost' "$f"; then - exit_code=1 - echo $f; cat "$f"; + echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." + echo $f + cat "$f" + exit 1 fi; done - exit $exit_code - k8s-netpol-e2e: name: Kubernetes Network Policy E2E if: | @@ -483,6 +493,9 @@ jobs: - name: Load image run: docker load --input kube-ovn.tar + - name: Export debug image tag + run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" + - name: Create kind cluster run: | sudo pip3 install j2cli @@ -492,6 +505,9 @@ jobs: sudo chown -R $(id -un). ~/.kube/ - name: Install Kube-OVN + env: + VERSION: ${{ env.DEBUG_TAG }} + DEBUG_WRAPPER: valgrind run: make kind-install-${{ matrix.ip-family }} - name: Run E2E @@ -511,6 +527,35 @@ jobs: name: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log path: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz + - name: Check valgrind result + run: | + kubectl -n kube-system rollout restart deploy ovn-central + kubectl -n kube-system rollout restart ds ovs-ovn + kubectl -n kube-system rollout status deploy ovn-central + kubectl -n kube-system rollout status ds ovs-ovn + while true; do + if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then + break + fi + sleep 1 + done + kubectl ko log ovn + kubectl ko log ovs + + for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do + echo "Checking if valgrind log file for $daemon exists..." + find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 + done + + find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do + if grep -qw 'definitely lost' "$f"; then + echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." + echo $f + cat "$f" + exit 1 + fi; + done + k8s-netpol-legacy-e2e: name: Kubernetes Network Policy Legacy E2E if: | @@ -592,6 +637,9 @@ jobs: - name: Load image run: docker load --input kube-ovn.tar + - name: Export debug image tag + run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" + - name: Create kind cluster run: | sudo pip3 install j2cli @@ -601,6 +649,9 @@ jobs: sudo chown -R $(id -un). ~/.kube/ - name: Install Kube-OVN + env: + VERSION: ${{ env.DEBUG_TAG }} + DEBUG_WRAPPER: valgrind run: make kind-install-${{ matrix.ip-family }} - name: Run E2E @@ -620,6 +671,35 @@ jobs: name: k8s-netpol-legacy-e2e-${{ matrix.ip-family }}-ko-log path: k8s-netpol-legacy-e2e-${{ matrix.ip-family }}-ko-log.tar.gz + - name: Check valgrind result + run: | + kubectl -n kube-system rollout restart deploy ovn-central + kubectl -n kube-system rollout restart ds ovs-ovn + kubectl -n kube-system rollout status deploy ovn-central + kubectl -n kube-system rollout status ds ovs-ovn + while true; do + if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then + break + fi + sleep 1 + done + kubectl ko log ovn + kubectl ko log ovs + + for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do + echo "Checking if valgrind log file for $daemon exists..." + find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 + done + + find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do + if grep -qw 'definitely lost' "$f"; then + echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." + echo $f + cat "$f" + exit 1 + fi; + done + cyclonus-netpol-e2e: name: Cyclonus Network Policy E2E if: | @@ -677,6 +757,9 @@ jobs: - name: Load image run: docker load --input kube-ovn.tar + - name: Export debug image tag + run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV" + - name: Create kind cluster run: | sudo pip3 install j2cli @@ -686,6 +769,9 @@ jobs: sudo chown -R $(id -un). ~/.kube/ - name: Install Kube-OVN + env: + VERSION: ${{ env.DEBUG_TAG }} + DEBUG_WRAPPER: valgrind run: make kind-install-${{ matrix.ip-family }} - name: Run E2E @@ -705,6 +791,35 @@ jobs: name: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log path: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz + - name: Check valgrind result + run: | + kubectl -n kube-system rollout restart deploy ovn-central + kubectl -n kube-system rollout restart ds ovs-ovn + kubectl -n kube-system rollout status deploy ovn-central + kubectl -n kube-system rollout status ds ovs-ovn + while true; do + if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then + break + fi + sleep 1 + done + kubectl ko log ovn + kubectl ko log ovs + + for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do + echo "Checking if valgrind log file for $daemon exists..." + find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 + done + + find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do + if grep -qw 'definitely lost' "$f"; then + echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." + echo $f + cat "$f" + exit 1 + fi; + done + kube-ovn-conformance-e2e: name: Kube-OVN Conformance E2E needs: @@ -824,19 +939,29 @@ jobs: kubectl -n kube-system rollout restart ds ovs-ovn kubectl -n kube-system rollout status deploy ovn-central kubectl -n kube-system rollout status ds ovs-ovn + while true; do + if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then + break + fi + sleep 1 + done kubectl ko log ovn kubectl ko log ovs - exit_code=0 - find kubectl-ko-log -type f -name '*.valgrind.*' | while read f; do + for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do + echo "Checking if valgrind log file for $daemon exists..." + find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1 + done + + find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do if grep -qw 'definitely lost' "$f"; then - exit_code=1 - echo $f; cat "$f"; + echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')." + echo $f + cat "$f" + exit 1 fi; done - exit $exit_code - - name: Cleanup run: sh dist/images/cleanup.sh