Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve functional tests #816

Merged
merged 1 commit into from
Oct 13, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ dist/*.docker
cover.out
.editorconfig
.idea/

bash_unit
34 changes: 31 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,18 @@ dist/flanneld: $(shell find . -type f -name '*.go')
-ldflags "-s -w -X github.com/coreos/flannel/version.Version=$(TAG)"

test: license-check gofmt
# Run the unit tests
go test -cover $(TEST_PACKAGES_EXPANDED)

# Test the docker-opts script
cd dist; ./mk-docker-opts_tests.sh

e2e-test: dist/flanneld-$(TAG)-$(ARCH).docker
cd dist; ./functional-test.sh $(REGISTRY):$(TAG)-$(ARCH)
# Run the functional tests
make e2e-test

e2e-test: bash_unit dist/flanneld-$(TAG)-$(ARCH).docker
./bash_unit dist/functional-test.sh
./bash_unit dist/functional-test-k8s.sh

cover:
# A single package must be given - e.g. 'PACKAGES=pkg/ip make cover'
Expand All @@ -74,6 +81,10 @@ update-glide:
# go get -d -u github.com/sgotti/glide-vc
glide vc --only-code --no-tests

bash_unit:
wget https://raw.githubusercontent.com/pgrange/bash_unit/v1.6.0/bash_unit
chmod +x bash_unit

clean:
rm -f dist/flanneld*
rm -f dist/*.aci
Expand Down Expand Up @@ -161,7 +172,7 @@ tar.gz:
tar -tvf dist/flannel-$(TAG)-linux-s390x.tar.gz

## Make a release after creating a tag
release: tar.gz
release: tar.gz release-tests
ARCH=amd64 make dist/flanneld-$(TAG)-amd64.aci
ARCH=arm make dist/flanneld-$(TAG)-arm.aci
ARCH=arm64 make dist/flanneld-$(TAG)-arm64.aci
Expand All @@ -171,6 +182,23 @@ release: tar.gz
@echo "Add all *.aci, flanneld-* and *.tar.gz files from dist/ to the Github release"
@echo "Use make docker-push-all to push the images to a registry"

release-tests: bash_unit
# Run the functional tests with different etcd versions.
ETCD_IMG="quay.io/coreos/etcd:latest" ./bash_unit dist/functional-test.sh
ETCD_IMG="quay.io/coreos/etcd:v3.2.7" ./bash_unit dist/functional-test.sh
ETCD_IMG="quay.io/coreos/etcd:v3.1.10" ./bash_unit dist/functional-test.sh
ETCD_IMG="quay.io/coreos/etcd:v3.0.17" ./bash_unit dist/functional-test.sh
# Etcd v2 docker image format is different. Override the etcd binary location so it works.
ETCD_IMG="quay.io/coreos/etcd:v2.3.8" ETCD_LOCATION=" " ./bash_unit dist/functional-test.sh

# Run the functional tests with different k8s versions. Currently these are the latest point releases.
# This list should be updated during the release process.
K8S_VERSION="1.7.6" ./bash_unit dist/functional-test-k8s.sh
K8S_VERSION="1.6.10" ./bash_unit dist/functional-test-k8s.sh
# K8S_VERSION="1.5.7" ./bash_unit dist/functional-test-k8s.sh #kube-flannel.yml is incompatible
# K8S_VERSION="1.4.12" ./bash_unit dist/functional-test-k8s.sh #kube-flannel.yml is incompatible
# K8S_VERSION="1.3.10" ./bash_unit dist/functional-test-k8s.sh #kube-flannel.yml is incompatible

docker-push-all:
ARCH=amd64 make docker-push
ARCH=arm make docker-push
Expand Down
121 changes: 121 additions & 0 deletions dist/functional-test-k8s.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
#!/bin/bash
ETCD_IMG="${ETCD_IMG:-quay.io/coreos/etcd:v3.2.7}"
ETCD_LOCATION="${ETCD_LOCATION:-etcd}"
FLANNEL_NET="${FLANNEL_NET:-10.10.0.0/16}"
TAG=`git describe --tags --dirty`
FLANNEL_DOCKER_IMAGE="${FLANNEL_DOCKER_IMAGE:-quay.io/coreos/flannel:$TAG}"
K8S_VERSION="${K8S_VERSION:-1.7.6}"

docker_ip=$(ip -o -f inet addr show docker0 | grep -Po 'inet \K[\d.]+')
etcd_endpt="http://$docker_ip:2379"
k8s_endpt="http://$docker_ip:8080"

setup_suite() {
# Run etcd, killing any existing one that was running

# Start etcd
docker rm -f flannel-e2e-test-etcd >/dev/null 2>/dev/null
docker run --name=flannel-e2e-test-etcd -d -p 2379:2379 $ETCD_IMG etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls $etcd_endpt >/dev/null
sleep 1

# Start a kubernetes API server
docker rm -f flannel-e2e-k8s-apiserver >/dev/null 2>/dev/null
docker run -d --net=host --name flannel-e2e-k8s-apiserver gcr.io/google_containers/hyperkube-amd64:v$K8S_VERSION \
/hyperkube apiserver --etcd-servers=$etcd_endpt \
--service-cluster-ip-range=10.101.0.0/16 --insecure-bind-address=0.0.0.0 --allow-privileged >/dev/null
sleep 1

while ! cat <<EOF | docker run -i --rm --net=host gcr.io/google_containers/hyperkube-amd64:v$K8S_VERSION /hyperkube kubectl create -f - >/dev/null 2>/dev/null
apiVersion: v1
kind: Node
metadata:
name: flannel1
annotations:
dummy: value
spec:
podCIDR: 10.10.1.0/24
EOF
do
sleep 1
done

cat <<EOF | docker run -i --rm --net=host gcr.io/google_containers/hyperkube-amd64:v$K8S_VERSION /hyperkube kubectl create -f - >/dev/null 2>/dev/null
apiVersion: v1
kind: Node
metadata:
name: flannel2
annotations:
dummy: value
spec:
podCIDR: 10.10.2.0/24
EOF
}

teardown_suite() {
# Teardown the etcd server
docker rm -f flannel-e2e-test-etcd >/dev/null
docker rm -f flannel-e2e-k8s-apiserver >/dev/null
}

teardown() {
docker rm -f flannel-e2e-test-flannel1 >/dev/null 2>/dev/null
docker rm -f flannel-e2e-test-flannel2 >/dev/null 2>/dev/null
}

start_flannel() {
local backend=$1

flannel_conf="{ \"Network\": \"$FLANNEL_NET\", \"Backend\": { \"Type\": \"${backend}\" } }"
for host_num in 1 2; do
docker rm -f flannel-e2e-test-flannel$host_num >/dev/null 2>/dev/null
docker run -e NODE_NAME=flannel$host_num --privileged --name flannel-e2e-test-flannel$host_num -tid --entrypoint /bin/sh $FLANNEL_DOCKER_IMAGE >/dev/null
docker exec flannel-e2e-test-flannel$host_num /bin/sh -c 'mkdir -p /etc/kube-flannel'
echo $flannel_conf | docker exec -i flannel-e2e-test-flannel$host_num /bin/sh -c 'cat > /etc/kube-flannel/net-conf.json'
docker exec -d flannel-e2e-test-flannel$host_num /opt/bin/flanneld --kube-subnet-mgr --kube-api-url $k8s_endpt
done
}

create_ping_dest() {
# add a dummy interface with $FLANNEL_SUBNET so we have a known working IP to ping
for host_num in 1 2; do
while ! docker exec flannel-e2e-test-flannel$host_num ls /run/flannel/subnet.env >/dev/null 2>&1; do
sleep 0.1
done

# Use declare to allow the host_num variable to be part of the ping_dest variable name. -g is needed to make it global
declare -g ping_dest$host_num=$(docker "exec" --privileged flannel-e2e-test-flannel$host_num /bin/sh -c '\
source /run/flannel/subnet.env && \
ip link add name dummy0 type dummy && \
ip addr add $FLANNEL_SUBNET dev dummy0 && ip link set dummy0 up && \
echo $FLANNEL_SUBNET | cut -f 1 -d "/" ')
done
}

test_vxlan() {
start_flannel vxlan
create_ping_dest # creates ping_dest1 and ping_dest2 variables
pings
}

test_udp() {
start_flannel udp
create_ping_dest # creates ping_dest1 and ping_dest2 variables
pings
}

test_host-gw() {
start_flannel host-gw
create_ping_dest # creates ping_dest1 and ping_dest2 variables
pings
}

pings() {
# ping in both directions
assert "docker exec -it --privileged flannel-e2e-test-flannel1 /bin/ping -c 5 $ping_dest2" "Host 1 cannot ping host 2"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or we could put this in some sort of retry block, maybe

assert "docker exec -it --privileged flannel-e2e-test-flannel2 /bin/ping -c 5 $ping_dest1" "Host 2 cannot ping host 1"
}

test_manifest() {
# This just tests that the API server accepts the manifest, not that it actually acts on it correctly.
assert "cat ../Documentation/kube-flannel.yml | docker run -i --rm --net=host gcr.io/google_containers/hyperkube-amd64:v$K8S_VERSION /hyperkube kubectl create -f -"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

btw, there's a cool project garethr/kubeval that can validate the k8s manifests. Not suggesting we should or shouldn't use it, but just throwing it out there

}
Loading