Skip to content

Commit

Permalink
ci: fix valgrind result analysis (#2853)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed May 29, 2023
1 parent e1b7919 commit 4d1b12a
Showing 1 changed file with 179 additions and 13 deletions.
192 changes: 179 additions & 13 deletions .github/workflows/build-x86-image.yaml
View file
Expand Up @@ -28,6 +28,8 @@ jobs:
build-kube-ovn-base:
name: Build kube-ovn-base
runs-on: ubuntu-22.04
outputs:
build-base: ${{ steps.build.outputs.build-base }}
steps:
- uses: actions/checkout@v3
with:
Expand Down Expand Up @@ -116,7 +118,7 @@ jobs:
go mod tidy
git diff --exit-code
make lint
if [ "x${{ env.BUILD_BASE }}" = "x1" ]; then
if [ ${{ needs.build-kube-ovn-base.outputs.build-base || 0 }} = 1 ]; then
TAG=$(cat VERSION)
docker tag kubeovn/kube-ovn-base:$TAG-amd64 kubeovn/kube-ovn-base:$TAG
docker tag kubeovn/kube-ovn-base:$TAG-amd64-no-avx512 kubeovn/kube-ovn-base:$TAG-no-avx512
Expand Down Expand Up @@ -369,19 +371,29 @@ jobs:
kubectl -n kube-system rollout status deploy ovn-central
kubectl -n kube-system delete po -l app=ovs
kubectl -n kube-system wait pod -l app=ovs --for condition=Ready --timeout=90s
while true; do
if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then
break
fi
sleep 1
done
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovn
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovs
exit_code=0
find kubectl-ko-log -type f -name '*.valgrind.*' | while read f; do
for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
echo "Checking if valgrind log file for $daemon exists..."
find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
done
find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
if grep -qw 'definitely lost' "$f"; then
exit_code=1
echo $f; cat "$f";
echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
echo $f
cat "$f"
exit 1
fi;
done
exit $exit_code
k8s-netpol-e2e:
name: Kubernetes Network Policy E2E
if: |
Expand Down Expand Up @@ -462,6 +474,9 @@ jobs:
- name: Load image
run: docker load --input kube-ovn.tar

- name: Export debug image tag
run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"

- name: Create kind cluster
run: |
sudo pip3 install j2cli
Expand All @@ -471,12 +486,57 @@ jobs:
sudo chown -R $(id -un). ~/.kube/
- name: Install Kube-OVN
env:
VERSION: ${{ env.DEBUG_TAG }}
DEBUG_WRAPPER: valgrind
run: make kind-install-${{ matrix.ip-family }}

- name: Run E2E
working-directory: ${{ env.E2E_DIR }}
run: make k8s-netpol-e2e

- name: kubectl ko log
if: failure()
run: |
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log all
mv kubectl-ko-log.tar.gz k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
- name: upload kubectl ko log
uses: actions/upload-artifact@v3
if: failure()
with:
name: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log
path: k8s-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

- name: Check valgrind result
run: |
kubectl -n kube-system rollout restart deploy ovn-central
kubectl -n kube-system rollout status deploy ovn-central
kubectl -n kube-system delete po -l app=ovs
kubectl -n kube-system wait pod -l app=ovs --for condition=Ready --timeout=90s
while true; do
if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then
break
fi
sleep 1
done
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovn
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovs
for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
echo "Checking if valgrind log file for $daemon exists..."
find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
done
find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
if grep -qw 'definitely lost' "$f"; then
echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
echo $f
cat "$f"
exit 1
fi;
done
k8s-netpol-legacy-e2e:
name: Kubernetes Network Policy Legacy E2E
if: |
Expand Down Expand Up @@ -557,6 +617,9 @@ jobs:
- name: Load image
run: docker load --input kube-ovn.tar

- name: Export debug image tag
run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"

- name: Create kind cluster
run: |
sudo pip3 install j2cli
Expand All @@ -566,12 +629,57 @@ jobs:
sudo chown -R $(id -un). ~/.kube/
- name: Install Kube-OVN
env:
VERSION: ${{ env.DEBUG_TAG }}
DEBUG_WRAPPER: valgrind
run: make kind-install-${{ matrix.ip-family }}

- name: Run E2E
working-directory: ${{ env.E2E_DIR }}
run: make k8s-netpol-legacy-e2e

- name: kubectl ko log
if: failure()
run: |
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log all
mv kubectl-ko-log.tar.gz k8s-netpol-legacy-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
- name: upload kubectl ko log
uses: actions/upload-artifact@v3
if: failure()
with:
name: k8s-netpol-legacy-e2e-${{ matrix.ip-family }}-ko-log
path: k8s-netpol-legacy-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

- name: Check valgrind result
run: |
kubectl -n kube-system rollout restart deploy ovn-central
kubectl -n kube-system rollout status deploy ovn-central
kubectl -n kube-system delete po -l app=ovs
kubectl -n kube-system wait pod -l app=ovs --for condition=Ready --timeout=90s
while true; do
if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then
break
fi
sleep 1
done
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovn
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovs
for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
echo "Checking if valgrind log file for $daemon exists..."
find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
done
find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
if grep -qw 'definitely lost' "$f"; then
echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
echo $f
cat "$f"
exit 1
fi;
done
cyclonus-netpol-e2e:
name: Cyclonus Network Policy E2E
if: |
Expand Down Expand Up @@ -629,6 +737,9 @@ jobs:
- name: Load image
run: docker load --input kube-ovn.tar

- name: Export debug image tag
run: echo "DEBUG_TAG='$(cat VERSION)-debug'" >> "$GITHUB_ENV"

- name: Create kind cluster
run: |
sudo pip3 install j2cli
Expand All @@ -638,12 +749,57 @@ jobs:
sudo chown -R $(id -un). ~/.kube/
- name: Install Kube-OVN
env:
VERSION: ${{ env.DEBUG_TAG }}
DEBUG_WRAPPER: valgrind
run: make kind-install-${{ matrix.ip-family }}

- name: Run E2E
working-directory: ${{ env.E2E_DIR }}
run: make cyclonus-netpol-e2e

- name: kubectl ko log
if: failure()
run: |
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log all
mv kubectl-ko-log.tar.gz cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
- name: upload kubectl ko log
uses: actions/upload-artifact@v3
if: failure()
with:
name: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log
path: cyclonus-netpol-e2e-${{ matrix.ip-family }}-ko-log.tar.gz

- name: Check valgrind result
run: |
kubectl -n kube-system rollout restart deploy ovn-central
kubectl -n kube-system rollout status deploy ovn-central
kubectl -n kube-system delete po -l app=ovs
kubectl -n kube-system wait pod -l app=ovs --for condition=Ready --timeout=90s
while true; do
if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then
break
fi
sleep 1
done
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovn
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovs
for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
echo "Checking if valgrind log file for $daemon exists..."
find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
done
find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
if grep -qw 'definitely lost' "$f"; then
echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
echo $f
cat "$f"
exit 1
fi;
done
kube-ovn-conformance-e2e:
name: Kube-OVN Conformance E2E
needs:
Expand Down Expand Up @@ -749,19 +905,29 @@ jobs:
kubectl -n kube-system rollout status deploy ovn-central
kubectl -n kube-system delete po -l app=ovs
kubectl -n kube-system wait pod -l app=ovs --for condition=Ready --timeout=90s
while true; do
if [ ! -z "$(kubectl -n kube-system get ep ovn-nb -o jsonpath='{.subsets}')" ]; then
break
fi
sleep 1
done
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovn
bash ${{ env.E2E_DIR }}/dist/images/kubectl-ko log ovs
exit_code=0
find kubectl-ko-log -type f -name '*.valgrind.*' | while read f; do
for daemon in ovsdb-nb ovsdb-sb ovn-northd ovn-controller ovsdb-server ovs-vswitchd; do
echo "Checking if valgrind log file for $daemon exists..."
find kubectl-ko-log -type f -name "$daemon.valgrind.log.[[:digit:]]*" -exec false {} + && exit 1
done
find kubectl-ko-log -type f -name '*.valgrind.log.*' | while read f; do
if grep -qw 'definitely lost' "$f"; then
exit_code=1
echo $f; cat "$f";
echo "Memory leak detected in $(basename $f | awk -F. '{print $1}')."
echo $f
cat "$f"
exit 1
fi;
done
exit $exit_code
- name: Cleanup
run: sh dist/images/cleanup.sh

Expand Down

0 comments on commit 4d1b12a

Please sign in to comment.