Skip to content

Commit

Permalink
Add node e2e test CI.
Browse files Browse the repository at this point in the history
Signed-off-by: Lantao Liu <lantaol@google.com>
  • Loading branch information
Random-Liu committed Aug 21, 2017
1 parent dcc3cb2 commit c05a7e7
Show file tree
Hide file tree
Showing 12 changed files with 165 additions and 78 deletions.
31 changes: 17 additions & 14 deletions .travis.yml
Expand Up @@ -22,33 +22,36 @@ before_script:

jobs:
include:
- stage: Verify
- stage: Build
script:
- make install.tools
- make .gitvalidation
- make verify
- make binaries
go: 1.8.x
- script:
- make install.tools
- make .gitvalidation
- make verify
- make binaries
go: tip
- stage: Build and Test
- stage: Test
script:
- make install.deps
- make test
- make binaries
- make test-cri
after_script:
# Abuse travis to preserve the log.
- cat /tmp/test-cri/cri-containerd.log
- cat /tmp/test-cri/containerd.log
go: 1.8.x
- script:
- stage: E2E Test
script:
- test "${TRAVIS_EVENT_TYPE}" != "cron" && exit 0
- make install.deps
- make test
- make binaries
- make test-cri
go: tip

after_script:
# Abuse travis to preserve the log.
# TODO(random-liu): Use prow for integration test.
- cat /tmp/cri-containerd.log
- cat /tmp/containerd.log
- make test-e2e-node
after_script:
# TODO(random-liu): Upload log to GCS.
- test "${TRAVIS_EVENT_TYPE}" != "cron" && exit 0
- cat /tmp/test-e2e-node/cri-containerd.log
go: 1.8.x
31 changes: 15 additions & 16 deletions Makefile
Expand Up @@ -24,6 +24,7 @@ VERSION := $(shell git describe --tags --dirty)
# strip the first char of the tag if it's a `v`
VERSION := $(VERSION:v%=%)
BUILD_TAGS:= -ldflags '-X $(PROJECT)/pkg/version.criContainerdVersion=$(VERSION)'
SOURCES := $(shell find . -name '*.go')

all: binaries

Expand All @@ -37,26 +38,20 @@ help:
@echo " * 'static-binaries - Build static cri-containerd"
@echo " * 'test' - Test cri-containerd"
@echo " * 'test-cri' - Test cri-containerd with cri validation test"
@echo " * 'test-e2e-node' - Test cri-containerd with Kubernetes node e2e test"
@echo " * 'clean' - Clean artifacts"
@echo " * 'verify' - Execute the source code verification tools"
@echo " * 'install.tools' - Install tools used by verify"
@echo " * 'install.deps' - Install dependencies of cri-containerd (containerd, runc, cni)"
@echo " * 'uninstall' - Remove installed binaries from system locations"
@echo " * 'version' - Print current cri-containerd release version"

.PHONY: check-gopath

check-gopath:
ifndef GOPATH
$(error GOPATH is not set)
endif

verify: lint gofmt boiler

version:
@echo $(VERSION)

lint: check-gopath
lint:
@echo "checking lint"
@./hack/verify-lint.sh

Expand All @@ -68,27 +63,30 @@ boiler:
@echo "checking boilerplate"
@./hack/verify-boilerplate.sh

cri-containerd: check-gopath
$(GO) build -o $(BUILD_DIR)/$@ \
$(BUILD_DIR)/cri-containerd: $(SOURCES)
$(GO) build -o $@ \
$(BUILD_TAGS) \
$(GO_LDFLAGS) $(GO_GCFLAGS) \
$(PROJECT)/cmd/cri-containerd

test:
go test -timeout=10m -race ./pkg/... $(BUILD_TAGS) $(GO_LDFLAGS) $(GO_GCFLAGS)

test-cri:
test-cri: binaries
@./hack/test-cri.sh

test-e2e-node: binaries
@./hack/test-e2e-node.sh

clean:
rm -f $(BUILD_DIR)/cri-containerd
rm -rf $(BUILD_DIR)/*

binaries: cri-containerd
binaries: $(BUILD_DIR)/cri-containerd

static-binaries: GO_LDFLAGS=--ldflags '-extldflags "-fno-PIC -static"'
static-binaries: cri-containerd
static-binaries: $(BUILD_DIR)/cri-containerd

install: check-gopath
install: binaries
install -D -m 755 $(BUILD_DIR)/cri-containerd $(BINDIR)/cri-containerd

uninstall:
Expand All @@ -102,7 +100,7 @@ install.deps:
.PHONY: .gitvalidation
# When this is running in travis, it will only check the travis commit range.
# When running outside travis, it will check from $(EPOCH_TEST_COMMIT)..HEAD.
.gitvalidation: check-gopath
.gitvalidation:
ifeq ($(TRAVIS),true)
git-validation -q -run DCO,short-subject
else
Expand Down Expand Up @@ -132,5 +130,6 @@ install.tools: .install.gitvalidation .install.gometalinter
lint \
test \
test-cri \
test-e2e-node \
uninstall \
version
42 changes: 6 additions & 36 deletions hack/test-cri.sh
Expand Up @@ -16,31 +16,20 @@
set -o nounset
set -o pipefail

ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/..
. ${ROOT}/hack/versions
source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh

# FOCUS focuses the test to run.
FOCUS=${FOCUS:-}
# SKIP skips the test to skip.
SKIP=${SKIP:-"attach|RunAsUser|host port"}
REPORT_DIR=${REPORT_DIR:-"/tmp"}
REPORT_DIR=${REPORT_DIR:-"/tmp/test-cri"}

if [[ -z "${GOPATH}" ]]; then
echo "GOPATH is not set"
exit 1
fi

if [[ ! "${PATH}" =~ (^|:)${GOPATH}/bin(|/)(:|$) ]]; then
echo "GOPATH/bin is not in path"
exit 1
fi

if [ ! -x ${ROOT}/_output/cri-containerd ]; then
echo "cri-containerd is not built"
exit 1
fi

CRITEST=critest
CRITEST=${GOPATH}/bin/critest
CRITEST_PKG=github.com/kubernetes-incubator/cri-tools
CRICONTAINERD_SOCK=/var/run/cri-containerd.sock

Expand All @@ -54,32 +43,13 @@ if [ ! -x "$(command -v ${CRITEST})" ]; then
fi
which ${CRITEST}

# Start containerd
if [ ! -x "$(command -v containerd)" ]; then
echo "containerd is not installed, please run hack/install-deps.sh"
exit 1
fi
sudo pkill containerd
sudo containerd -l debug &> ${REPORT_DIR}/containerd.log &

# Wait for containerd to be running by using the containerd client ctr to check the version
# of the containerd server. Wait an increasing amount of time after each of five attempts
MAX_ATTEMPTS=5
attempt_num=1
until sudo ctr version &> /dev/null || (( attempt_num == MAX_ATTEMPTS ))
do
echo "Attempt $attempt_num to connect to containerd failed! Trying again in $attempt_num seconds..."
sleep $(( attempt_num++ ))
done

# Start cri-containerd
cd ${ROOT}
sudo _output/cri-containerd --alsologtostderr --v 4 &> ${REPORT_DIR}/cri-containerd.log &
mkdir -p ${REPORT_DIR}
start_cri_containerd ${REPORT_DIR}

# Run cri validation test
sudo env PATH=${PATH} GOPATH=${GOPATH} ${CRITEST} --runtime-endpoint=${CRICONTAINERD_SOCK} --focus="${FOCUS}" --ginkgo-flags="--skip=\"${SKIP}\"" validation
test_exit_code=$?

sudo pkill containerd
kill_cri_containerd

exit ${test_exit_code}
61 changes: 61 additions & 0 deletions hack/test-e2e-node.sh
@@ -0,0 +1,61 @@
#!/bin/bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o nounset
set -o pipefail

source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh

DEFAULT_SKIP="\[Flaky\]|\[Slow\]|\[Serial\]"
DEFAULT_SKIP+="|runAsUser"
DEFAULT_SKIP+="|scheduling\sa\sGuaranteed\sPod"
DEFAULT_SKIP+="|scheduling\sa\sBurstable\sPod"
DEFAULT_SKIP+="|AllowPrivilegeEscalation"
DEFAULT_SKIP+="|scheduling\sa\sBestEffort\sPod"
DEFAULT_SKIP+="|querying\s\/stats\/summary"
DEFAULT_SKIP+="|set\sto\sthe\smanifest\sdigest"
DEFAULT_SKIP+="|AppArmor"
DEFAULT_SKIP+="|Top\slevel\sQoS\scontainers"
DEFAULT_SKIP+="|pull\sfrom\sprivate\sregistry\swith\ssecret"

# FOCUS focuses the test to run.
export FOCUS=${FOCUS:-""}
# SKIP skips the test to skip.
export SKIP=${SKIP:-${DEFAULT_SKIP}}
REPORT_DIR=${REPORT_DIR:-"/tmp/test-e2e-node"}

if [[ -z "${GOPATH}" ]]; then
echo "GOPATH is not set"
exit 1
fi

# Get kubernetes
KUBERNETES_REPO="https://github.com/kubernetes/kubernetes"
KUBERNETES_PATH="${GOPATH}/src/k8s.io/kubernetes"
if [ ! -d "${KUBERNETES_PATH}" ]; then
mkdir -p ${KUBERNETES_PATH}
cd ${KUBERNETES_PATH}
git clone ${KUBERNETES_REPO} .
fi
cd ${KUBERNETES_PATH}
git fetch --all
git checkout ${KUBERNETES_VERSION}

mkdir -p ${REPORT_DIR}
start_cri_containerd ${REPORT_DIR}

make test-e2e-node RUNTIME=remote CONTAINER_RUNTIME_ENDPOINT=unix:///var/run/cri-containerd.sock ARTIFACTS=${REPORT_DIR}

kill_cri_containerd
53 changes: 53 additions & 0 deletions hack/test-utils.sh
@@ -0,0 +1,53 @@
#!/bin/bash

# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/..
. ${ROOT}/hack/versions

# start_cri_containerd starts containerd and cri-containerd.
start_cri_containerd() {
local report_dir=$1
if [ ! -x ${ROOT}/_output/cri-containerd ]; then
echo "cri-containerd is not built"
exit 1
fi

# Start containerd
if [ ! -x "$(command -v containerd)" ]; then
echo "containerd is not installed, please run hack/install-deps.sh"
exit 1
fi
kill_cri_containerd
sudo containerd -l debug &> ${report_dir}/containerd.log &

# Wait for containerd to be running by using the containerd client ctr to check the version
# of the containerd server. Wait an increasing amount of time after each of five attempts
local MAX_ATTEMPTS=5
local attempt_num=1
until sudo ctr version &> /dev/null || (( attempt_num == MAX_ATTEMPTS ))
do
echo "attempt $attempt_num to connect to containerd failed! Trying again in $attempt_num seconds..."
sleep $(( attempt_num++ ))
done

# Start cri-containerd
sudo ${ROOT}/_output/cri-containerd --alsologtostderr --v 4 &> ${report_dir}/cri-containerd.log &
}

# kill_cri_containerd kills containerd and cri-containerd.
kill_cri_containerd() {
sudo pkill containerd
}
1 change: 1 addition & 0 deletions hack/versions
Expand Up @@ -2,3 +2,4 @@ RUNC_VERSION=e775f0fba3ea329b8b766451c892c41a3d49594d
CNI_VERSION=v0.4.0
CONTAINERD_VERSION=938810e706bbcdbcb937ce63ba3e7c9ca329af64
CRITEST_VERSION=74bbd4e142f752f13c648d9dde23defed3e472a2
KUBERNETES_VERSION=d779e9c9561b732adf06263c5424889e7564fdbd
4 changes: 2 additions & 2 deletions pkg/server/container_list.go
Expand Up @@ -27,10 +27,10 @@ import (

// ListContainers lists all containers matching the filter.
func (c *criContainerdService) ListContainers(ctx context.Context, r *runtime.ListContainersRequest) (retRes *runtime.ListContainersResponse, retErr error) {
glog.V(4).Infof("ListContainers with filter %+v", r.GetFilter())
glog.V(5).Infof("ListContainers with filter %+v", r.GetFilter())
defer func() {
if retErr == nil {
glog.V(4).Infof("ListContainers returns containers %+v", retRes.GetContainers())
glog.V(5).Infof("ListContainers returns containers %+v", retRes.GetContainers())
}
}()

Expand Down
4 changes: 2 additions & 2 deletions pkg/server/container_status.go
Expand Up @@ -28,10 +28,10 @@ import (

// ContainerStatus inspects the container and returns the status.
func (c *criContainerdService) ContainerStatus(ctx context.Context, r *runtime.ContainerStatusRequest) (retRes *runtime.ContainerStatusResponse, retErr error) {
glog.V(4).Infof("ContainerStatus for container %q", r.GetContainerId())
glog.V(5).Infof("ContainerStatus for container %q", r.GetContainerId())
defer func() {
if retErr == nil {
glog.V(4).Infof("ContainerStatus for %q returns status %+v", r.GetContainerId(), retRes.GetStatus())
glog.V(5).Infof("ContainerStatus for %q returns status %+v", r.GetContainerId(), retRes.GetStatus())
}
}()

Expand Down
4 changes: 2 additions & 2 deletions pkg/server/image_list.go
Expand Up @@ -28,10 +28,10 @@ import (
// TODO(random-liu): Add image list filters after CRI defines this more clear, and kubelet
// actually needs it.
func (c *criContainerdService) ListImages(ctx context.Context, r *runtime.ListImagesRequest) (retRes *runtime.ListImagesResponse, retErr error) {
glog.V(4).Infof("ListImages with filter %+v", r.GetFilter())
glog.V(5).Infof("ListImages with filter %+v", r.GetFilter())
defer func() {
if retErr == nil {
glog.V(4).Infof("ListImages returns image list %+v", retRes.GetImages())
glog.V(5).Infof("ListImages returns image list %+v", retRes.GetImages())
}
}()

Expand Down
4 changes: 2 additions & 2 deletions pkg/server/image_status.go
Expand Up @@ -28,10 +28,10 @@ import (
// TODO(random-liu): We should change CRI to distinguish image id and image spec. (See
// kubernetes/kubernetes#46255)
func (c *criContainerdService) ImageStatus(ctx context.Context, r *runtime.ImageStatusRequest) (retRes *runtime.ImageStatusResponse, retErr error) {
glog.V(4).Infof("ImageStatus for image %q", r.GetImage().GetImage())
glog.V(5).Infof("ImageStatus for image %q", r.GetImage().GetImage())
defer func() {
if retErr == nil {
glog.V(4).Infof("ImageStatus for %q returns image status %+v",
glog.V(5).Infof("ImageStatus for %q returns image status %+v",
r.GetImage().GetImage(), retRes.GetImage())
}
}()
Expand Down
4 changes: 2 additions & 2 deletions pkg/server/sandbox_list.go
Expand Up @@ -30,10 +30,10 @@ import (

// ListPodSandbox returns a list of Sandbox.
func (c *criContainerdService) ListPodSandbox(ctx context.Context, r *runtime.ListPodSandboxRequest) (retRes *runtime.ListPodSandboxResponse, retErr error) {
glog.V(4).Infof("ListPodSandbox with filter %+v", r.GetFilter())
glog.V(5).Infof("ListPodSandbox with filter %+v", r.GetFilter())
defer func() {
if retErr == nil {
glog.V(4).Infof("ListPodSandbox returns sandboxes %+v", retRes.GetItems())
glog.V(5).Infof("ListPodSandbox returns sandboxes %+v", retRes.GetItems())
}
}()

Expand Down

0 comments on commit c05a7e7

Please sign in to comment.