diff --git a/.travis.yml b/.travis.yml index d98a721e198a..5be98101c9e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,33 +22,36 @@ before_script: jobs: include: - - stage: Verify + - stage: Build script: - make install.tools - make .gitvalidation - make verify + - make binaries go: 1.8.x - script: - make install.tools - make .gitvalidation - make verify + - make binaries go: tip - - stage: Build and Test + - stage: Test script: - make install.deps - make test - - make binaries - make test-cri + after_script: + # Abuse travis to preserve the log. + - cat /tmp/test-cri/cri-containerd.log + - cat /tmp/test-cri/containerd.log go: 1.8.x - - script: + - stage: E2E Test + script: + - test "${TRAVIS_EVENT_TYPE}" != "cron" && exit 0 - make install.deps - - make test - - make binaries - - make test-cri - go: tip - -after_script: - # Abuse travis to preserve the log. - # TODO(random-liu): Use prow for integration test. - - cat /tmp/cri-containerd.log - - cat /tmp/containerd.log + - make test-e2e-node + after_script: + # TODO(random-liu): Upload log to GCS. + - test "${TRAVIS_EVENT_TYPE}" != "cron" && exit 0 + - cat /tmp/test-e2e-node/cri-containerd.log + go: 1.8.x diff --git a/Makefile b/Makefile index 0a9dcd590fea..8e622861467f 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ VERSION := $(shell git describe --tags --dirty) # strip the first char of the tag if it's a `v` VERSION := $(VERSION:v%=%) BUILD_TAGS:= -ldflags '-X $(PROJECT)/pkg/version.criContainerdVersion=$(VERSION)' +SOURCES := $(shell find . -name '*.go') all: binaries @@ -37,6 +38,7 @@ help: @echo " * 'static-binaries - Build static cri-containerd" @echo " * 'test' - Test cri-containerd" @echo " * 'test-cri' - Test cri-containerd with cri validation test" + @echo " * 'test-e2e-node' - Test cri-containerd with Kubernetes node e2e test" @echo " * 'clean' - Clean artifacts" @echo " * 'verify' - Execute the source code verification tools" @echo " * 'install.tools' - Install tools used by verify" @@ -44,19 +46,12 @@ help: @echo " * 'uninstall' - Remove installed binaries from system locations" @echo " * 'version' - Print current cri-containerd release version" -.PHONY: check-gopath - -check-gopath: -ifndef GOPATH - $(error GOPATH is not set) -endif - verify: lint gofmt boiler version: @echo $(VERSION) -lint: check-gopath +lint: @echo "checking lint" @./hack/verify-lint.sh @@ -68,8 +63,8 @@ boiler: @echo "checking boilerplate" @./hack/verify-boilerplate.sh -cri-containerd: check-gopath - $(GO) build -o $(BUILD_DIR)/$@ \ +$(BUILD_DIR)/cri-containerd: $(SOURCES) + $(GO) build -o $@ \ $(BUILD_TAGS) \ $(GO_LDFLAGS) $(GO_GCFLAGS) \ $(PROJECT)/cmd/cri-containerd @@ -77,18 +72,21 @@ cri-containerd: check-gopath test: go test -timeout=10m -race ./pkg/... $(BUILD_TAGS) $(GO_LDFLAGS) $(GO_GCFLAGS) -test-cri: +test-cri: binaries @./hack/test-cri.sh +test-e2e-node: binaries + @./hack/test-e2e-node.sh + clean: - rm -f $(BUILD_DIR)/cri-containerd + rm -rf $(BUILD_DIR)/* -binaries: cri-containerd +binaries: $(BUILD_DIR)/cri-containerd static-binaries: GO_LDFLAGS=--ldflags '-extldflags "-fno-PIC -static"' -static-binaries: cri-containerd +static-binaries: $(BUILD_DIR)/cri-containerd -install: check-gopath +install: binaries install -D -m 755 $(BUILD_DIR)/cri-containerd $(BINDIR)/cri-containerd uninstall: @@ -102,7 +100,7 @@ install.deps: .PHONY: .gitvalidation # When this is running in travis, it will only check the travis commit range. # When running outside travis, it will check from $(EPOCH_TEST_COMMIT)..HEAD. -.gitvalidation: check-gopath +.gitvalidation: ifeq ($(TRAVIS),true) git-validation -q -run DCO,short-subject else @@ -132,5 +130,6 @@ install.tools: .install.gitvalidation .install.gometalinter lint \ test \ test-cri \ + test-e2e-node \ uninstall \ version diff --git a/hack/test-cri.sh b/hack/test-cri.sh index 44dae91993e1..63a217b123d4 100755 --- a/hack/test-cri.sh +++ b/hack/test-cri.sh @@ -16,31 +16,20 @@ set -o nounset set -o pipefail -ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/.. -. ${ROOT}/hack/versions +source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh # FOCUS focuses the test to run. FOCUS=${FOCUS:-} # SKIP skips the test to skip. SKIP=${SKIP:-"attach|RunAsUser|host port"} -REPORT_DIR=${REPORT_DIR:-"/tmp"} +REPORT_DIR=${REPORT_DIR:-"/tmp/test-cri"} if [[ -z "${GOPATH}" ]]; then echo "GOPATH is not set" exit 1 fi -if [[ ! "${PATH}" =~ (^|:)${GOPATH}/bin(|/)(:|$) ]]; then - echo "GOPATH/bin is not in path" - exit 1 -fi - -if [ ! -x ${ROOT}/_output/cri-containerd ]; then - echo "cri-containerd is not built" - exit 1 -fi - -CRITEST=critest +CRITEST=${GOPATH}/bin/critest CRITEST_PKG=github.com/kubernetes-incubator/cri-tools CRICONTAINERD_SOCK=/var/run/cri-containerd.sock @@ -54,32 +43,13 @@ if [ ! -x "$(command -v ${CRITEST})" ]; then fi which ${CRITEST} -# Start containerd -if [ ! -x "$(command -v containerd)" ]; then - echo "containerd is not installed, please run hack/install-deps.sh" - exit 1 -fi -sudo pkill containerd -sudo containerd -l debug &> ${REPORT_DIR}/containerd.log & - -# Wait for containerd to be running by using the containerd client ctr to check the version -# of the containerd server. Wait an increasing amount of time after each of five attempts -MAX_ATTEMPTS=5 -attempt_num=1 -until sudo ctr version &> /dev/null || (( attempt_num == MAX_ATTEMPTS )) -do - echo "Attempt $attempt_num to connect to containerd failed! Trying again in $attempt_num seconds..." - sleep $(( attempt_num++ )) -done - -# Start cri-containerd -cd ${ROOT} -sudo _output/cri-containerd --alsologtostderr --v 4 &> ${REPORT_DIR}/cri-containerd.log & +mkdir -p ${REPORT_DIR} +start_cri_containerd ${REPORT_DIR} # Run cri validation test sudo env PATH=${PATH} GOPATH=${GOPATH} ${CRITEST} --runtime-endpoint=${CRICONTAINERD_SOCK} --focus="${FOCUS}" --ginkgo-flags="--skip=\"${SKIP}\"" validation test_exit_code=$? -sudo pkill containerd +kill_cri_containerd exit ${test_exit_code} diff --git a/hack/test-e2e-node.sh b/hack/test-e2e-node.sh new file mode 100755 index 000000000000..a55528507d0b --- /dev/null +++ b/hack/test-e2e-node.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -o nounset +set -o pipefail + +source $(dirname "${BASH_SOURCE[0]}")/test-utils.sh + +DEFAULT_SKIP="\[Flaky\]|\[Slow\]|\[Serial\]" +DEFAULT_SKIP+="|runAsUser" +DEFAULT_SKIP+="|scheduling\sa\sGuaranteed\sPod" +DEFAULT_SKIP+="|scheduling\sa\sBurstable\sPod" +DEFAULT_SKIP+="|AllowPrivilegeEscalation" +DEFAULT_SKIP+="|scheduling\sa\sBestEffort\sPod" +DEFAULT_SKIP+="|querying\s\/stats\/summary" +DEFAULT_SKIP+="|set\sto\sthe\smanifest\sdigest" +DEFAULT_SKIP+="|AppArmor" +DEFAULT_SKIP+="|Top\slevel\sQoS\scontainers" +DEFAULT_SKIP+="|pull\sfrom\sprivate\sregistry\swith\ssecret" + +# FOCUS focuses the test to run. +export FOCUS=${FOCUS:-""} +# SKIP skips the test to skip. +export SKIP=${SKIP:-${DEFAULT_SKIP}} +REPORT_DIR=${REPORT_DIR:-"/tmp/test-e2e-node"} + +if [[ -z "${GOPATH}" ]]; then + echo "GOPATH is not set" + exit 1 +fi + +# Get kubernetes +KUBERNETES_REPO="https://github.com/kubernetes/kubernetes" +KUBERNETES_PATH="${GOPATH}/src/k8s.io/kubernetes" +if [ ! -d "${KUBERNETES_PATH}" ]; then + mkdir -p ${KUBERNETES_PATH} + cd ${KUBERNETES_PATH} + git clone ${KUBERNETES_REPO} . +fi +cd ${KUBERNETES_PATH} +git fetch --all +git checkout ${KUBERNETES_VERSION} + +mkdir -p ${REPORT_DIR} +start_cri_containerd ${REPORT_DIR} + +make test-e2e-node RUNTIME=remote CONTAINER_RUNTIME_ENDPOINT=unix:///var/run/cri-containerd.sock ARTIFACTS=${REPORT_DIR} + +kill_cri_containerd diff --git a/hack/test-utils.sh b/hack/test-utils.sh new file mode 100644 index 000000000000..8104f6cc11f1 --- /dev/null +++ b/hack/test-utils.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"/.. +. ${ROOT}/hack/versions + +# start_cri_containerd starts containerd and cri-containerd. +start_cri_containerd() { + local report_dir=$1 + if [ ! -x ${ROOT}/_output/cri-containerd ]; then + echo "cri-containerd is not built" + exit 1 + fi + + # Start containerd + if [ ! -x "$(command -v containerd)" ]; then + echo "containerd is not installed, please run hack/install-deps.sh" + exit 1 + fi + kill_cri_containerd + sudo containerd -l debug &> ${report_dir}/containerd.log & + + # Wait for containerd to be running by using the containerd client ctr to check the version + # of the containerd server. Wait an increasing amount of time after each of five attempts + local MAX_ATTEMPTS=5 + local attempt_num=1 + until sudo ctr version &> /dev/null || (( attempt_num == MAX_ATTEMPTS )) + do + echo "attempt $attempt_num to connect to containerd failed! Trying again in $attempt_num seconds..." + sleep $(( attempt_num++ )) + done + + # Start cri-containerd + sudo ${ROOT}/_output/cri-containerd --alsologtostderr --v 4 &> ${report_dir}/cri-containerd.log & +} + +# kill_cri_containerd kills containerd and cri-containerd. +kill_cri_containerd() { + sudo pkill containerd +} diff --git a/hack/versions b/hack/versions index 3f6b5f530a45..20de8e11bfad 100644 --- a/hack/versions +++ b/hack/versions @@ -2,3 +2,4 @@ RUNC_VERSION=e775f0fba3ea329b8b766451c892c41a3d49594d CNI_VERSION=v0.4.0 CONTAINERD_VERSION=938810e706bbcdbcb937ce63ba3e7c9ca329af64 CRITEST_VERSION=74bbd4e142f752f13c648d9dde23defed3e472a2 +KUBERNETES_VERSION=d779e9c9561b732adf06263c5424889e7564fdbd diff --git a/pkg/server/container_list.go b/pkg/server/container_list.go index d0f587657ac6..b34fd7286ac0 100644 --- a/pkg/server/container_list.go +++ b/pkg/server/container_list.go @@ -27,10 +27,10 @@ import ( // ListContainers lists all containers matching the filter. func (c *criContainerdService) ListContainers(ctx context.Context, r *runtime.ListContainersRequest) (retRes *runtime.ListContainersResponse, retErr error) { - glog.V(4).Infof("ListContainers with filter %+v", r.GetFilter()) + glog.V(5).Infof("ListContainers with filter %+v", r.GetFilter()) defer func() { if retErr == nil { - glog.V(4).Infof("ListContainers returns containers %+v", retRes.GetContainers()) + glog.V(5).Infof("ListContainers returns containers %+v", retRes.GetContainers()) } }() diff --git a/pkg/server/container_status.go b/pkg/server/container_status.go index aa7dca8eb483..600a240e3e10 100644 --- a/pkg/server/container_status.go +++ b/pkg/server/container_status.go @@ -28,10 +28,10 @@ import ( // ContainerStatus inspects the container and returns the status. func (c *criContainerdService) ContainerStatus(ctx context.Context, r *runtime.ContainerStatusRequest) (retRes *runtime.ContainerStatusResponse, retErr error) { - glog.V(4).Infof("ContainerStatus for container %q", r.GetContainerId()) + glog.V(5).Infof("ContainerStatus for container %q", r.GetContainerId()) defer func() { if retErr == nil { - glog.V(4).Infof("ContainerStatus for %q returns status %+v", r.GetContainerId(), retRes.GetStatus()) + glog.V(5).Infof("ContainerStatus for %q returns status %+v", r.GetContainerId(), retRes.GetStatus()) } }() diff --git a/pkg/server/image_list.go b/pkg/server/image_list.go index dc80d98ecdad..8e936e5e01f9 100644 --- a/pkg/server/image_list.go +++ b/pkg/server/image_list.go @@ -28,10 +28,10 @@ import ( // TODO(random-liu): Add image list filters after CRI defines this more clear, and kubelet // actually needs it. func (c *criContainerdService) ListImages(ctx context.Context, r *runtime.ListImagesRequest) (retRes *runtime.ListImagesResponse, retErr error) { - glog.V(4).Infof("ListImages with filter %+v", r.GetFilter()) + glog.V(5).Infof("ListImages with filter %+v", r.GetFilter()) defer func() { if retErr == nil { - glog.V(4).Infof("ListImages returns image list %+v", retRes.GetImages()) + glog.V(5).Infof("ListImages returns image list %+v", retRes.GetImages()) } }() diff --git a/pkg/server/image_status.go b/pkg/server/image_status.go index f93462291861..2e7ce5427d79 100644 --- a/pkg/server/image_status.go +++ b/pkg/server/image_status.go @@ -28,10 +28,10 @@ import ( // TODO(random-liu): We should change CRI to distinguish image id and image spec. (See // kubernetes/kubernetes#46255) func (c *criContainerdService) ImageStatus(ctx context.Context, r *runtime.ImageStatusRequest) (retRes *runtime.ImageStatusResponse, retErr error) { - glog.V(4).Infof("ImageStatus for image %q", r.GetImage().GetImage()) + glog.V(5).Infof("ImageStatus for image %q", r.GetImage().GetImage()) defer func() { if retErr == nil { - glog.V(4).Infof("ImageStatus for %q returns image status %+v", + glog.V(5).Infof("ImageStatus for %q returns image status %+v", r.GetImage().GetImage(), retRes.GetImage()) } }() diff --git a/pkg/server/sandbox_list.go b/pkg/server/sandbox_list.go index cccf6fea8b78..a859129c2870 100644 --- a/pkg/server/sandbox_list.go +++ b/pkg/server/sandbox_list.go @@ -30,10 +30,10 @@ import ( // ListPodSandbox returns a list of Sandbox. func (c *criContainerdService) ListPodSandbox(ctx context.Context, r *runtime.ListPodSandboxRequest) (retRes *runtime.ListPodSandboxResponse, retErr error) { - glog.V(4).Infof("ListPodSandbox with filter %+v", r.GetFilter()) + glog.V(5).Infof("ListPodSandbox with filter %+v", r.GetFilter()) defer func() { if retErr == nil { - glog.V(4).Infof("ListPodSandbox returns sandboxes %+v", retRes.GetItems()) + glog.V(5).Infof("ListPodSandbox returns sandboxes %+v", retRes.GetItems()) } }() diff --git a/pkg/server/sandbox_status.go b/pkg/server/sandbox_status.go index 600fa7217ca7..7f1e63415e74 100644 --- a/pkg/server/sandbox_status.go +++ b/pkg/server/sandbox_status.go @@ -31,10 +31,10 @@ import ( // PodSandboxStatus returns the status of the PodSandbox. func (c *criContainerdService) PodSandboxStatus(ctx context.Context, r *runtime.PodSandboxStatusRequest) (retRes *runtime.PodSandboxStatusResponse, retErr error) { - glog.V(4).Infof("PodSandboxStatus for sandbox %q", r.GetPodSandboxId()) + glog.V(5).Infof("PodSandboxStatus for sandbox %q", r.GetPodSandboxId()) defer func() { if retErr == nil { - glog.V(4).Infof("PodSandboxStatus for %q returns status %+v", r.GetPodSandboxId(), retRes.GetStatus()) + glog.V(5).Infof("PodSandboxStatus for %q returns status %+v", r.GetPodSandboxId(), retRes.GetStatus()) } }()