From 353206c9e016199d0a21bd52a93c71f3e33d513c Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Fri, 23 Apr 2021 10:25:00 -0700 Subject: [PATCH 1/2] Add Cyclonus network policy tests --- .github/workflows/cyclonus-netpol-test.yaml | 54 +++++++++++++++++++++ Makefile | 11 ++++- README.md | 2 +- cns/azure-cns.yaml | 2 +- npm/azure-npm.yaml | 10 ++-- test/cyclonus/install-cyclonus.yaml | 28 +++++++++++ test/cyclonus/test-cyclonus.sh | 43 ++++++++++++++++ test/kind/kind.yaml | 13 +++++ 8 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/cyclonus-netpol-test.yaml create mode 100644 test/cyclonus/install-cyclonus.yaml create mode 100755 test/cyclonus/test-cyclonus.sh create mode 100644 test/kind/kind.yaml diff --git a/.github/workflows/cyclonus-netpol-test.yaml b/.github/workflows/cyclonus-netpol-test.yaml new file mode 100644 index 0000000000..19577e3ad4 --- /dev/null +++ b/.github/workflows/cyclonus-netpol-test.yaml @@ -0,0 +1,54 @@ +name: Cyclonus Network Policy Test + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + schedule: + # run once a day at midnight + - cron: '0 0 * * *' + +jobs: + cyclonus-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Make NPM image + run: | + VERSION=cyclonus make azure-npm-image + + - name: Setup Kind + uses: engineerd/setup-kind@v0.5.0 + with: + version: "v0.9.0" + config: ./test/kind/kind.yaml + name: npm-kind + + - name: Install Azure NPM + run: | + sed -i 's/mcr.microsoft.com\/containernetworking\/azure-npm:v1.3.1/acnpublic.azurecr.io\/azure-npm:cyclonus/' ./npm/azure-npm.yaml + kind load docker-image acnpublic.azurecr.io/azure-npm:cyclonus --name npm-kind + kubectl apply -f ./npm/azure-npm.yaml + + - name: Run Cyclonus network policy test + run: make test-cyclonus + + - name: Fetch logs + if: always() + run: | + kubectl logs -n kube-system -l k8s-app=azure-npm --tail -1 --prefix > npm-logs.txt + mv ./test/cyclonus/cyclonus-test.txt ./cyclonus-test.txt + + - name: 'Upload Logs' + uses: actions/upload-artifact@v2 + if: always() + with: + name: logs + path: | + ./npm-logs.txt + ./cyclonus-test.txt + diff --git a/Makefile b/Makefile index 8d9e8ba4ae..114651b013 100644 --- a/Makefile +++ b/Makefile @@ -461,4 +461,13 @@ test-all: # run all tests .PHONY: test-integration test-integration: - go test -coverpkg=./... -v -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration... \ No newline at end of file + go test -coverpkg=./... -v -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration... + +.PHONY: test-cyclonus +test-cyclonus: + cd test/cyclonus && bash ./test-cyclonus.sh + cd .. + +.PHONY: kind +kind: + kind create cluster --config ./test/kind/kind.yaml \ No newline at end of file diff --git a/README.md b/README.md index b7b1d0e5f7..46a7a74b68 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Microsoft Azure Container Networking [![Build Status](https://msazure.visualstudio.com/One/_apis/build/status/Custom/Networking/ContainerNetworking/Azure.azure-container-networking?branchName=master)](https://msazure.visualstudio.com/One/_build/latest?definitionId=95007&branchName=master) [![Go Report Card](https://goreportcard.com/badge/github.com/Azure/azure-container-networking)](https://goreportcard.com/report/github.com/Azure/azure-container-networking) ![GitHub release](https://img.shields.io/github/release/Azure/azure-container-networking.svg) -[![codecov](https://codecov.io/gh/Azure/azure-container-networking/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/azure-container-networking) + ## Overview This repository contains container networking services and plugins for Linux and Windows containers running on Azure: diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index a009556480..e88c8c46e4 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -93,7 +93,7 @@ spec: effect: NoSchedule containers: - name: cns-container - image: mcr.microsoft.com/containernetworking/azure-cns:v1.2.4 + image: mcr.microsoft.com/containernetworking/azure-cns:v1.2.9 imagePullPolicy: IfNotPresent args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] volumeMounts: diff --git a/npm/azure-npm.yaml b/npm/azure-npm.yaml index 3d74afcb22..d273cdaadf 100644 --- a/npm/azure-npm.yaml +++ b/npm/azure-npm.yaml @@ -6,7 +6,7 @@ metadata: labels: addonmanager.kubernetes.io/mode: EnsureExists --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: azure-npm @@ -33,7 +33,7 @@ rules: - list - watch --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: azure-npm-binding @@ -77,12 +77,10 @@ spec: effect: NoSchedule - key: CriticalAddonsOnly operator: Exists - nodeSelector: - beta.kubernetes.io/os: linux - kubernetes.io/role: agent + containers: - name: azure-npm - image: mcr.microsoft.com/containernetworking/azure-npm:v1.1.5 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.3.1 resources: limits: cpu: 250m diff --git a/test/cyclonus/install-cyclonus.yaml b/test/cyclonus/install-cyclonus.yaml new file mode 100644 index 0000000000..c21928b1b1 --- /dev/null +++ b/test/cyclonus/install-cyclonus.yaml @@ -0,0 +1,28 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: cyclonus + namespace: kube-system + labels: + app: cyclonus +spec: + template: + spec: + restartPolicy: Never + containers: + - command: + - ./cyclonus + - generate + - --noisy=true + - --ignore-loopback=true + - --cleanup-namespaces=true + - --perturbation-wait-seconds=5 + - --pod-creation-timeout-seconds=20 + - --job-timeout-seconds=2 + - --server-protocol=TCP,UDP + - --server-port=80 + name: cyclonus + imagePullPolicy: IfNotPresent + image: mfenwick100/cyclonus:v0.4.4 + serviceAccount: cyclonus + \ No newline at end of file diff --git a/test/cyclonus/test-cyclonus.sh b/test/cyclonus/test-cyclonus.sh new file mode 100755 index 0000000000..1343766cc1 --- /dev/null +++ b/test/cyclonus/test-cyclonus.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -eo pipefail +set -xv + +kubectl delete --ignore-not-found=true clusterrolebinding cyclonus +kubectl delete --ignore-not-found=true sa cyclonus -n kube-system +kubectl delete --ignore-not-found=true -f ./install-cyclonus.yaml +kubectl delete --ignore-not-found=true ns x y z + +sleep 5 + +# set up cyclonus +kubectl create clusterrolebinding cyclonus --clusterrole=cluster-admin --serviceaccount=kube-system:cyclonus +kubectl create sa cyclonus -n kube-system +kubectl create -f ./install-cyclonus.yaml + +time kubectl wait --for=condition=ready --timeout=1m pod -n kube-system -l job-name=cyclonus + +#!/bin/bash +{ kubectl logs -f -n kube-system job.batch/cyclonus; } & +{ time kubectl wait --for=condition=completed --timeout=600m pod -n kube-system -l job-name=cyclonus; } & +wait -n +pkill -P $$ +echo done + +# grab the job logs +LOG_FILE=cyclonus-test.txt +kubectl logs -n kube-system job.batch/cyclonus | tee "$LOG_FILE" +cat "$LOG_FILE" + +kubectl delete --ignore-not-found=true clusterrolebinding cyclonus +kubectl delete --ignore-not-found=true sa cyclonus -n kube-system +kubectl delete --ignore-not-found=true -f ./install-cyclonus.yaml + +# if 'failure' is in the logs, fail; otherwise succeed +rc=0 + +cat "$LOG_FILE" | grep "failed" > /dev/null 2>&1 || rc=$? +echo $rc +if [ $rc -eq 0 ]; then + exit 1 +fi \ No newline at end of file diff --git a/test/kind/kind.yaml b/test/kind/kind.yaml new file mode 100644 index 0000000000..0c323fd4c6 --- /dev/null +++ b/test/kind/kind.yaml @@ -0,0 +1,13 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + image: kindest/node:v1.19.1 + - role: worker + image: kindest/node:v1.19.1 + - role: worker + image: kindest/node:v1.19.1 +networking: + ipFamily: ipv4 + podSubnet: "10.10.0.0/16" + serviceSubnet: "10.11.0.0/16" From 30930f5f7b7ec87058f9c0a9202da988f5d68b34 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Tue, 4 May 2021 17:04:16 -0700 Subject: [PATCH 2/2] update readme --- Makefile | 3 ++- README.md | 6 ++++++ cns/azure-cns.yaml | 2 +- test/cyclonus/test-cyclonus.sh | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 114651b013..0439d618b4 100644 --- a/Makefile +++ b/Makefile @@ -470,4 +470,5 @@ test-cyclonus: .PHONY: kind kind: - kind create cluster --config ./test/kind/kind.yaml \ No newline at end of file + kind create cluster --config ./test/kind/kind.yaml + \ No newline at end of file diff --git a/README.md b/README.md index 46a7a74b68..f1007af8c7 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ [![Build Status](https://msazure.visualstudio.com/One/_apis/build/status/Custom/Networking/ContainerNetworking/Azure.azure-container-networking?branchName=master)](https://msazure.visualstudio.com/One/_build/latest?definitionId=95007&branchName=master) [![Go Report Card](https://goreportcard.com/badge/github.com/Azure/azure-container-networking)](https://goreportcard.com/report/github.com/Azure/azure-container-networking) ![GitHub release](https://img.shields.io/github/release/Azure/azure-container-networking.svg) +| Azure Network Policy Manager Conformance | | +| ----------- | ----------- | +| Cyclonus Network Policy Suite | [![Cyclonus Network Policy Test](https://github.com/Azure/azure-container-networking/actions/workflows/cyclonus-netpol-test.yaml/badge.svg?branch=master)](https://github.com/Azure/azure-container-networking/actions/workflows/cyclonus-netpol-test.yaml) | +| Kubernetes Network Policy E2E | [![Build Status](https://dev.azure.com/msazure/One/_apis/build/status/Custom/Networking/ContainerNetworking/NPM%20Conformance%20Tests?branchName=master)](https://dev.azure.com/msazure/One/_build/latest?definitionId=195725&branchName=master) | + + ## Overview This repository contains container networking services and plugins for Linux and Windows containers running on Azure: diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index e88c8c46e4..a009556480 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -93,7 +93,7 @@ spec: effect: NoSchedule containers: - name: cns-container - image: mcr.microsoft.com/containernetworking/azure-cns:v1.2.9 + image: mcr.microsoft.com/containernetworking/azure-cns:v1.2.4 imagePullPolicy: IfNotPresent args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] volumeMounts: diff --git a/test/cyclonus/test-cyclonus.sh b/test/cyclonus/test-cyclonus.sh index 1343766cc1..5ce106edf6 100755 --- a/test/cyclonus/test-cyclonus.sh +++ b/test/cyclonus/test-cyclonus.sh @@ -40,4 +40,4 @@ cat "$LOG_FILE" | grep "failed" > /dev/null 2>&1 || rc=$? echo $rc if [ $rc -eq 0 ]; then exit 1 -fi \ No newline at end of file +fi