diff --git a/.travis.yml b/.travis.yml index 15e1479e6c7d..9acfce780bfc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,27 +35,15 @@ jobs: - make install.tools - make .gitvalidation - make binaries - - make containerd go: "1.10" - stage: Test script: - make install.deps + - make containerd + - sudo make install-containerd - make test - make test-integration - make test-cri - after_script: - # Abuse travis to preserve the log. - - cat /tmp/test-integration/cri-containerd.log - - cat /tmp/test-integration/containerd.log - - cat /tmp/test-cri/cri-containerd.log - - cat /tmp/test-cri/containerd.log - go: "1.10" - - script: - - make install.deps - - make containerd - - sudo make install-containerd - - make test-integration STANDALONE_CRI_CONTAINERD=false - - make test-cri STANDALONE_CRI_CONTAINERD=false after_script: # Abuse travis to preserve the log. - cat /tmp/test-integration/containerd.log diff --git a/Makefile b/Makefile index 8b1f75eb182c..2da939e8faa3 100644 --- a/Makefile +++ b/Makefile @@ -41,12 +41,14 @@ help: @echo "Usage: make " @echo @echo " * 'install' - Install binaries to system locations" - @echo " * 'binaries' - Build cri-containerd and ctrcri" - @echo " * 'static-binaries - Build static cri-containerd and ctrcri" + @echo " * 'binaries' - Build containerd and ctrcri" + @echo " * 'static-binaries - Build static containerd and ctrcri" + @echo " * 'ctrcri' - Build ctrcri" + @echo " * 'install-ctrcri' - Install ctrcri" + @echo " * 'containerd' - Build a customized containerd with CRI plugin for testing" + @echo " * 'install-containerd' - Install customized containerd to system location" @echo " * 'release' - Build release tarball" @echo " * 'push' - Push release tarball to GCS" - @echo " * 'containerd' - Build a customized containerd with CRI plugin for testing" - @echo " * 'install-containerd' - Install customized containerd to system location" @echo " * 'test' - Test cri-containerd with unit test" @echo " * 'test-integration' - Test cri-containerd with integration test" @echo " * 'test-cri' - Test cri-containerd with cri validation test" @@ -89,13 +91,6 @@ sync-vendor: update-vendor: sync-vendor sort-vendor -$(BUILD_DIR)/cri-containerd: $(SOURCES) - $(GO) build -o $@ \ - -tags '$(BUILD_TAGS)' \ - -ldflags '$(GO_LDFLAGS)' \ - -gcflags '$(GO_GCFLAGS)' \ - $(PROJECT)/cmd/cri-containerd - $(BUILD_DIR)/ctrcri: $(SOURCES) $(GO) build -o $@ \ -tags '$(BUILD_TAGS)' \ @@ -131,17 +126,25 @@ test-e2e-node: binaries clean: rm -rf $(BUILD_DIR)/* -binaries: $(BUILD_DIR)/cri-containerd $(BUILD_DIR)/ctrcri +binaries: $(BUILD_DIR)/containerd $(BUILD_DIR)/ctrcri static-binaries: GO_LDFLAGS += -extldflags "-fno-PIC -static" -static-binaries: $(BUILD_DIR)/cri-containerd $(BUILD_DIR)/ctrcri +static-binaries: $(BUILD_DIR)/containerd $(BUILD_DIR)/ctrcri + +ctrcri: $(BUILD_DIR)/ctrcri -install: binaries - install -D -m 755 $(BUILD_DIR)/cri-containerd $(BINDIR)/cri-containerd +install-ctrcri: ctrcri install -D -m 755 $(BUILD_DIR)/ctrcri $(BINDIR)/ctrcri +containerd: $(BUILD_DIR)/containerd + +install-containerd: containerd + install -D -m 755 $(BUILD_DIR)/containerd $(BINDIR)/containerd + +install: install-ctrcri install-containerd + uninstall: - rm -f $(BINDIR)/cri-containerd + rm -f $(BINDIR)/containerd rm -f $(BINDIR)/ctrcri $(BUILD_DIR)/$(TARBALL): static-binaries vendor.conf @@ -152,11 +155,6 @@ release: $(BUILD_DIR)/$(TARBALL) push: $(BUILD_DIR)/$(TARBALL) @BUILD_DIR=$(BUILD_DIR) TARBALL=$(TARBALL) VERSION=$(VERSION) ./hack/push.sh -containerd: $(BUILD_DIR)/containerd - -install-containerd: containerd - install -D -m 755 $(BUILD_DIR)/containerd $(BINDIR)/containerd - proto: @hack/update-proto.sh @@ -189,6 +187,10 @@ install.tools: .install.gitvalidation .install.gometalinter .PHONY: \ binaries \ static-binaries \ + ctrcri \ + install-ctrcri \ + containerd \ + install-containerd \ release \ push \ boiler \ @@ -198,8 +200,6 @@ install.tools: .install.gitvalidation .install.gometalinter help \ install \ lint \ - containerd \ - install-containerd \ test \ test-integration \ test-cri \ diff --git a/cluster/gce/cloud-init/master.yaml b/cluster/gce/cloud-init/master.yaml index d6dc1bcba03e..c2a2c9704ca3 100644 --- a/cluster/gce/cloud-init/master.yaml +++ b/cluster/gce/cloud-init/master.yaml @@ -1,29 +1,31 @@ #cloud-config write_files: -# Setup cri-containerd. - - path: /etc/systemd/system/cri-containerd-installation.service +# Setup containerd. + - path: /etc/systemd/system/containerd-installation.service permissions: 0644 owner: root content: | # installed by cloud-init [Unit] - Description=Download and install cri-containerd binaries and configurations. + Description=Download and install containerd binaries and configurations. After=network-online.target [Service] Type=oneshot RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /home/cri-containerd - ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd - ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd - ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh - ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh - ExecStart=/home/cri-containerd/configure.sh + ExecStartPre=/bin/mkdir -p /home/containerd + ExecStartPre=/bin/mount --bind /home/containerd /home/containerd + ExecStartPre=/bin/mount -o remount,exec /home/containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh + ExecStart=/home/containerd/configure.sh [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target + # containerd on master uses the cni binary and config in the + # release tarball. - path: /etc/containerd/config.toml permissions: 0644 owner: root @@ -35,8 +37,14 @@ write_files: path = "/runtime" [plugins.linux] - shim = "/home/cri-containerd/usr/local/bin/containerd-shim" - runtime = "/home/cri-containerd/usr/local/sbin/runc" + shim = "/home/containerd/usr/local/bin/containerd-shim" + runtime = "/home/containerd/usr/local/sbin/runc" + + [plugins.cri.cni] + bin_dir = "/home/containerd/opt/cni/bin" + conf_dir = "/home/containerd/etc/cni/net.d" + [plugins.cri.registry.mirrors."docker.io"] + endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] - path: /etc/systemd/system/containerd.service permissions: 0644 @@ -46,7 +54,7 @@ write_files: [Unit] Description=containerd container runtime Documentation=https://containerd.io - After=cri-containerd-installation.service + After=containerd-installation.service [Service] Restart=always @@ -59,67 +67,36 @@ write_files: LimitNPROC=infinity LimitCORE=infinity ExecStartPre=/sbin/modprobe overlay - ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug - - [Install] - WantedBy=cri-containerd.target - - - path: /etc/systemd/system/cri-containerd.service - permissions: 0644 - owner: root - content: | - # installed by cloud-init - [Unit] - Description=Kubernetes containerd CRI shim - Requires=network-online.target - After=cri-containerd-installation.service - - [Service] - Restart=always - RestartSec=5 - LimitNOFILE=1048576 - # Having non-zero Limit*s causes performance problems due to accounting overhead - # in the kernel. We recommend using cgroups to do container-local accounting. - LimitNPROC=infinity - LimitCORE=infinity - # cri-containerd on master uses the cni binary and config in the - # release tarball. - ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ - --log-level=debug \ - --network-bin-dir=/home/cri-containerd/opt/cni/bin \ - --network-conf-dir=/home/cri-containerd/etc/cni/net.d \ - --cgroup-path=/runtime \ - --registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io + ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - - path: /etc/systemd/system/cri-containerd-monitor.service + - path: /etc/systemd/system/containerd-monitor.service permissions: 0644 owner: root content: | [Unit] - Description=Kubernetes health monitoring for cri-containerd and containerd - After=containerd.service cri-containerd.service + Description=Kubernetes health monitoring for containerd + After=containerd.service [Service] Restart=always RestartSec=10 RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh - ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ - /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' + ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh + ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \ + /home/containerd/opt/containerd/cluster/health-monitor.sh' [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - # TODO(random-liu): Guarantee order. - - path: /etc/systemd/system/cri-containerd.target + - path: /etc/systemd/system/containerd.target permissions: 0644 owner: root content: | [Unit] - Description=CRI Containerd + Description=Containerd [Install] WantedBy=kubernetes.target @@ -221,11 +198,10 @@ write_files: runcmd: - systemctl daemon-reload + - systemctl enable containerd-installation.service - systemctl enable containerd.service - - systemctl enable cri-containerd-installation.service - - systemctl enable cri-containerd.service - - systemctl enable cri-containerd-monitor.service - - systemctl enable cri-containerd.target + - systemctl enable containerd-monitor.service + - systemctl enable containerd.target - systemctl enable kube-master-installation.service - systemctl enable kube-master-configuration.service - systemctl enable kubelet-monitor.service diff --git a/cluster/gce/cloud-init/node.yaml b/cluster/gce/cloud-init/node.yaml index c562279cf71f..6504861fd95c 100644 --- a/cluster/gce/cloud-init/node.yaml +++ b/cluster/gce/cloud-init/node.yaml @@ -1,28 +1,28 @@ #cloud-config write_files: -# Setup cri-containerd. - - path: /etc/systemd/system/cri-containerd-installation.service +# Setup containerd. + - path: /etc/systemd/system/containerd-installation.service permissions: 0644 owner: root content: | # installed by cloud-init [Unit] - Description=Download and install cri-containerd binaries and configurations. + Description=Download and install containerd binaries and configurations. After=network-online.target [Service] Type=oneshot RemainAfterExit=yes - ExecStartPre=/bin/mkdir -p /home/cri-containerd - ExecStartPre=/bin/mount --bind /home/cri-containerd /home/cri-containerd - ExecStartPre=/bin/mount -o remount,exec /home/cri-containerd - ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/cri-containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/cri-containerd-configure-sh - ExecStartPre=/bin/chmod 544 /home/cri-containerd/configure.sh - ExecStart=/home/cri-containerd/configure.sh + ExecStartPre=/bin/mkdir -p /home/containerd + ExecStartPre=/bin/mount --bind /home/containerd /home/containerd + ExecStartPre=/bin/mount -o remount,exec /home/containerd + ExecStartPre=/usr/bin/curl --fail --retry 5 --retry-delay 3 --silent --show-error -H "X-Google-Metadata-Request: True" -o /home/containerd/configure.sh http://metadata.google.internal/computeMetadata/v1/instance/attributes/containerd-configure-sh + ExecStartPre=/bin/chmod 544 /home/containerd/configure.sh + ExecStart=/home/containerd/configure.sh [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - path: /etc/containerd/config.toml permissions: 0644 @@ -35,8 +35,14 @@ write_files: path = "/runtime" [plugins.linux] - shim = "/home/cri-containerd/usr/local/bin/containerd-shim" - runtime = "/home/cri-containerd/usr/local/sbin/runc" + shim = "/home/containerd/usr/local/bin/containerd-shim" + runtime = "/home/containerd/usr/local/sbin/runc" + + [plugins.cri.cni] + bin_dir = "/home/kubernetes/bin" + conf_dir = "/etc/cni/net.d" + [plugins.cri.registry.mirrors."docker.io"] + endpoint = ["https://mirror.gcr.io","https://registry-1.docker.io"] - path: /etc/systemd/system/containerd.service permissions: 0644 @@ -46,7 +52,7 @@ write_files: [Unit] Description=containerd container runtime Documentation=https://containerd.io - After=cri-containerd-installation.service + After=containerd-installation.service [Service] Restart=always @@ -59,66 +65,36 @@ write_files: LimitNPROC=infinity LimitCORE=infinity ExecStartPre=/sbin/modprobe overlay - ExecStart=/home/cri-containerd/usr/local/bin/containerd --log-level debug - - [Install] - WantedBy=cri-containerd.target - - - path: /etc/systemd/system/cri-containerd.service - permissions: 0644 - owner: root - content: | - # installed by cloud-init - [Unit] - Description=Kubernetes containerd CRI shim - Requires=network-online.target - After=cri-containerd-installation.service - - [Service] - Restart=always - RestartSec=5 - LimitNOFILE=1048576 - # Having non-zero Limit*s causes performance problems due to accounting overhead - # in the kernel. We recommend using cgroups to do container-local accounting. - LimitNPROC=infinity - LimitCORE=infinity - # Point to /home/kubernetes/bin where calico setup cni binary in kube-up.sh. - # Point to /etc/cni/net.d where calico put cni config in kube-up.sh. - ExecStart=/home/cri-containerd/usr/local/bin/cri-containerd \ - --log-level=debug \ - --network-bin-dir=/home/kubernetes/bin \ - --network-conf-dir=/etc/cni/net.d \ - --cgroup-path=/runtime \ - --registry=docker.io=https://mirror.gcr.io,https://registry-1.docker.io + ExecStart=/home/containerd/usr/local/bin/containerd --log-level debug [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - - path: /etc/systemd/system/cri-containerd-monitor.service + - path: /etc/systemd/system/containerd-monitor.service permissions: 0644 owner: root content: | [Unit] - Description=Kubernetes health monitoring for cri-containerd and containerd - After=containerd.service cri-containerd.service + Description=Kubernetes health monitoring for containerd + After=containerd.service [Service] Restart=always RestartSec=10 RemainAfterExit=yes - ExecStartPre=/bin/chmod 544 /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh - ExecStart=/bin/bash -c 'CRICTL=/home/cri-containerd/usr/local/bin/crictl \ - /home/cri-containerd/opt/cri-containerd/cluster/health-monitor.sh' + ExecStartPre=/bin/chmod 544 /home/containerd/opt/containerd/cluster/health-monitor.sh + ExecStart=/bin/bash -c 'CRICTL=/home/containerd/usr/local/bin/crictl \ + /home/containerd/opt/containerd/cluster/health-monitor.sh' [Install] - WantedBy=cri-containerd.target + WantedBy=containerd.target - - path: /etc/systemd/system/cri-containerd.target + - path: /etc/systemd/system/containerd.target permissions: 0644 owner: root content: | [Unit] - Description=CRI Containerd + Description=Containerd [Install] WantedBy=kubernetes.target @@ -220,11 +196,10 @@ write_files: runcmd: - systemctl daemon-reload + - systemctl enable containerd-installation.service - systemctl enable containerd.service - - systemctl enable cri-containerd-installation.service - - systemctl enable cri-containerd.service - - systemctl enable cri-containerd-monitor.service - - systemctl enable cri-containerd.target + - systemctl enable containerd-monitor.service + - systemctl enable containerd.target - systemctl enable kube-node-installation.service - systemctl enable kube-node-configuration.service - systemctl enable kubelet-monitor.service diff --git a/cluster/gce/configure.sh b/cluster/gce/configure.sh index 9e3aba47ae0c..ca990ea8a239 100755 --- a/cluster/gce/configure.sh +++ b/cluster/gce/configure.sh @@ -19,9 +19,9 @@ set -o errexit set -o nounset set -o pipefail -# CRI_CONTAINERD_HOME is the directory for cri-containerd. -CRI_CONTAINERD_HOME="/home/cri-containerd" -cd "${CRI_CONTAINERD_HOME}" +# CONTAINERD_HOME is the directory for containerd. +CONTAINERD_HOME="/home/containerd" +cd "${CONTAINERD_HOME}" # fetch_metadata fetches metadata from GCE metadata server. # Var set: @@ -63,5 +63,5 @@ tar xvf "${TARBALL}" # Copy crictl config. cp "${CRI_CONTAINERD_HOME}/etc/crictl.yaml" /etc -echo "export PATH=${CRI_CONTAINERD_HOME}/usr/local/bin/:${CRI_CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ - /etc/profile.d/cri-containerd_env.sh +echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ + /etc/profile.d/containerd_env.sh diff --git a/cluster/gce/env b/cluster/gce/env index 184d86042012..231d1c914de7 100644 --- a/cluster/gce/env +++ b/cluster/gce/env @@ -9,11 +9,11 @@ if [ -z "${CRI_CONTAINERD_VERSION:-}" ]; then fi version_file=$(mktemp /tmp/version.XXXX) echo "${CRI_CONTAINERD_VERSION}" > "$version_file" -export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" -export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,cri-containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_MASTER_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/master.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" +export KUBE_NODE_EXTRA_METADATA="user-data=${GCE_DIR}/cloud-init/node.yaml,containerd-configure-sh=${GCE_DIR}/configure.sh,version=${version_file}" export KUBE_CONTAINER_RUNTIME="remote" -export KUBE_CONTAINER_RUNTIME_ENDPOINT="/var/run/cri-containerd.sock" -export KUBE_LOAD_IMAGE_COMMAND="/home/cri-containerd/usr/local/bin/ctrcri load" +export KUBE_CONTAINER_RUNTIME_ENDPOINT="/run/containerd/containerd.sock" +export KUBE_LOAD_IMAGE_COMMAND="/home/containerd/usr/local/bin/ctrcri load" export NETWORK_POLICY_PROVIDER="calico" export NON_MASQUERADE_CIDR="0.0.0.0/0" export KUBE_KUBELET_EXTRA_ARGS="--runtime-cgroups=/runtime" diff --git a/cluster/health-monitor.sh b/cluster/health-monitor.sh index 0320a534d7ed..800f4df07f22 100755 --- a/cluster/health-monitor.sh +++ b/cluster/health-monitor.sh @@ -20,16 +20,15 @@ set -o pipefail # CRICTL is the path of crictl CRICTL=${CRICTL:-"crictl"} # INITIAL_WAIT_ATTEMPTS is the number to attempt, before start -# performing health check. The problem is that cri-containerd -# and containerd are started around the same time with health -# monitor, they may not be ready yet when health-monitor is started. +# performing health check. The problem is that containerd is +# started around the same time with health monitor, it may +# not be ready yet when health-monitor is started. INITIAL_WAIT_ATTEMPTS=${INITIAL_WAIT_ATTEMPTS:-5} # COMMAND_TIMEOUT is the timeout for the health check command. COMMAND_TIMEOUT=${COMMAND_TIMEOUT:-60} # CHECK_PERIOD is the health check period. CHECK_PERIOD=${CHECK_PERIOD:-10} -# SLEEP_SECONDS is the time to sleep after killing cri-containerd -# and containerd. +# SLEEP_SECONDS is the time to sleep after killing containerd. SLEEP_SECONDS=${SLEEP_SECONDS:-120} attempt=1 @@ -41,11 +40,8 @@ done echo "Start performing health check." while true; do - # Use crictl pods because it requires both containerd and - # cri-containerd to be working. if ! timeout ${COMMAND_TIMEOUT} ${CRICTL} pods > /dev/null; then echo "\"$CRICTL pods\" failed!" - pkill -x cri-containerd pkill -x containerd # Wait for a while, as we don't want to kill it again before it is really up. sleep ${SLEEP_SECONDS} diff --git a/cmd/cri-containerd/cri_containerd.go b/cmd/cri-containerd/cri_containerd.go deleted file mode 100644 index 9b40cda19efe..000000000000 --- a/cmd/cri-containerd/cri_containerd.go +++ /dev/null @@ -1,229 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "flag" - "fmt" - "net" - "net/http" - "net/http/pprof" - "os" - "os/signal" - "runtime" - "syscall" - - "github.com/containerd/cgroups" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/sys" - runtimespec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "k8s.io/kubernetes/pkg/util/interrupt" - - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" - "github.com/containerd/cri-containerd/pkg/server" - "github.com/containerd/cri-containerd/pkg/version" -) - -// Add \u200B to avoid the space trimming. -const desc = "\u200B" + ` _ __ _ __ - __________(_) _________ ____ / /_____ _(_)____ ___ _________/ / - / ___/ ___/ /______/ ___/ __ \/ __ \/ __/ __ ` + "`" + `/ // __ \/ _ \/ ___/ __ / -/ /__/ / / //_____/ /__/ /_/ / / / / /_/ /_/ / // / / / __/ / / /_/ / -\___/_/ /_/ \___/\____/_/ /_/\__/\__,_/_//_/ /_/\___/_/ \__,_/ - -A containerd based Kubernetes CRI implementation. -` - -var cmd = &cobra.Command{ - Use: "cri-containerd", - Short: "A containerd based Kubernetes CRI implementation.", - Long: desc, -} - -func defaultConfigCommand() *cobra.Command { - return &cobra.Command{ - Use: "default-config", - Short: "Print default toml config of cri-containerd.", - Run: func(cmd *cobra.Command, args []string) { - options.PrintDefaultTomlConfig() - }, - } -} - -func versionCommand() *cobra.Command { - return &cobra.Command{ - Use: "version", - Short: "Print cri-containerd version information.", - Run: func(cmd *cobra.Command, args []string) { - version.PrintVersion() - }, - } -} - -func main() { - o := options.NewCRIContainerdOptions() - - o.AddFlags(cmd.Flags()) - cmd.AddCommand(defaultConfigCommand()) - cmd.AddCommand(versionCommand()) - - cmd.RunE = func(cmd *cobra.Command, args []string) error { - setupDumpStacksTrap() - if err := o.InitFlags(cmd.Flags()); err != nil { - return fmt.Errorf("failed to init CRI containerd flags: %v", err) - } - - if err := setLogLevel(o.LogLevel); err != nil { - return fmt.Errorf("failed to set log level: %v", err) - } - - logrus.Infof("Run cri-containerd %+v", o) - - if o.CgroupPath != "" { - _, err := loadCgroup(o.CgroupPath) - if err != nil { - return fmt.Errorf("failed to load cgroup for cgroup path %v: %v", o.CgroupPath, err) - } - } - - if o.OOMScore != 0 { - if err := sys.SetOOMScore(os.Getpid(), o.OOMScore); err != nil { - return fmt.Errorf("failed to set OOMScore to %v: %v", o.OOMScore, err) - } - } - - // Start profiling server if enable. - if o.EnableProfiling { - logrus.Info("Start profiling server") - go startProfilingServer(o.ProfilingAddress, o.ProfilingPort) - } - - logrus.Infof("Run cri-containerd grpc server on socket %q", o.SocketPath) - s, err := server.NewCRIContainerdService(o.Config) - if err != nil { - return fmt.Errorf("failed to create CRI containerd service: %v", err) - } - // Use interrupt handler to make sure the server is stopped properly. - // Pass in non-empty final function to avoid os.Exit(1). We expect `Run` - // to return itself. - h := interrupt.New(func(os.Signal) {}, func() { - if err := s.Close(); err != nil { - logrus.WithError(err).Error("Failed to stop cri service") - } - }) - if err := h.Run(func() error { return s.Run(true) }); err != nil { - return fmt.Errorf("failed to run cri-containerd with grpc server: %v", err) - } - return nil - } - - if err := cmd.Execute(); err != nil { - // Error should have been reported. - os.Exit(1) - } -} - -func setupDumpStacksTrap() { - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGUSR1) - go func() { - for range c { - dumpStacks() - } - }() -} - -func dumpStacks() { - buf := make([]byte, 1024) - for { - n := runtime.Stack(buf, true) - if n < len(buf) { - buf = buf[:n] - break - } - buf = make([]byte, 2*len(buf)) - } - logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf) -} - -// startProfilingServer start http server to profiling via web interface -func startProfilingServer(host string, port string) { - endpoint := net.JoinHostPort(host, port) - mux := http.NewServeMux() - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - if err := http.ListenAndServe(endpoint, mux); err != nil { - logrus.WithError(err).Error("Failed to start profiling server") - } -} - -func setLogLevel(l string) error { - lvl, err := log.ParseLevel(l) - if err != nil { - return err - } - if err := setGLogLevel(lvl); err != nil { - return err - } - logrus.SetLevel(lvl) - return nil -} - -// TODO(random-liu): Set glog level in plugin mode. -func setGLogLevel(l logrus.Level) error { - if err := flag.Set("logtostderr", "true"); err != nil { - return err - } - switch l { - case log.TraceLevel: - return flag.Set("v", "5") - case logrus.DebugLevel: - return flag.Set("v", "4") - case logrus.InfoLevel: - return flag.Set("v", "2") - // glog doesn't support following filters. Defaults to v=0. - case logrus.WarnLevel: - case logrus.ErrorLevel: - case logrus.FatalLevel: - case logrus.PanicLevel: - } - return nil -} - -// loadCgroup loads the cgroup associated with path if it exists and moves the current process into the cgroup. If the cgroup -// is not created it is created and returned. -func loadCgroup(cgroupPath string) (cgroups.Cgroup, error) { - cg, err := cgroups.Load(cgroups.V1, cgroups.StaticPath(cgroupPath)) - if err != nil { - if err != cgroups.ErrCgroupDeleted { - return nil, err - } - if cg, err = cgroups.New(cgroups.V1, cgroups.StaticPath(cgroupPath), &runtimespec.LinuxResources{}); err != nil { - return nil, err - } - } - if err := cg.Add(cgroups.Process{ - Pid: os.Getpid(), - }); err != nil { - return nil, err - } - return cg, nil -} diff --git a/cmd/cri-containerd/options/options.go b/cmd/cri-containerd/options/options.go deleted file mode 100644 index 3f1edfecb354..000000000000 --- a/cmd/cri-containerd/options/options.go +++ /dev/null @@ -1,273 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "os" - - "github.com/BurntSushi/toml" - "github.com/containerd/containerd" - "github.com/spf13/pflag" -) - -const ( - // configFilePathArgName is the path to the config file. - configFilePathArgName = "config" - // defaultConfigFilePath is the default config file path. - defaultConfigFilePath = "/etc/cri-containerd/config.toml" -) - -// ContainerdConfig contains toml config related to containerd -type ContainerdConfig struct { - // RootDir is the root directory path for containerd. - // TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode. - RootDir string `toml:"root_dir" json:"rootDir,omitempty"` - // Snapshotter is the snapshotter used by containerd. - Snapshotter string `toml:"snapshotter" json:"snapshotter,omitempty"` - // Endpoint is the containerd endpoint path. - // TODO(random-liu): Remove this field when no longer support cri-containerd standalone mode. - Endpoint string `toml:"endpoint" json:"endpoint,omitempty"` - // Runtime is the runtime to use in containerd. We may support - // other runtimes in the future. - Runtime string `toml:"runtime" json:"runtime,omitempty"` - // RuntimeEngine is the name of the runtime engine used by containerd. - // Containerd default should be "runc" - // We may support other runtime engines in the future. - RuntimeEngine string `toml:"runtime_engine" json:"runtimeEngine,omitempty"` - // RuntimeRoot is the directory used by containerd for runtime state. - // Containerd default should be "/run/containerd/runc" - RuntimeRoot string `toml:"runtime_root" json:"runtimeRoot,omitempty"` -} - -// CniConfig contains toml config related to cni -type CniConfig struct { - // NetworkPluginBinDir is the directory in which the binaries for the plugin is kept. - NetworkPluginBinDir string `toml:"bin_dir" json:"binDir,omitempty"` - // NetworkPluginConfDir is the directory in which the admin places a CNI conf. - NetworkPluginConfDir string `toml:"conf_dir" json:"confDir,omitempty"` -} - -// PluginConfig contains toml config related to CRI plugin, -// it is a subset of Config. -type PluginConfig struct { - // ContainerdConfig contains config related to containerd - ContainerdConfig `toml:"containerd" json:"containerd,omitempty"` - // CniConfig contains config related to cni - CniConfig `toml:"cni" json:"cni,omitempty"` - // Registry contains config related to the registry - Registry `toml:"registry" json:"registry,omitempty"` - // StreamServerAddress is the ip address streaming server is listening on. - StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress,omitempty"` - // StreamServerPort is the port streaming server is listening on. - StreamServerPort string `toml:"stream_server_port" json:"streamServerPort,omitempty"` - // EnableSelinux indicates to enable the selinux support. - EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux,omitempty"` - // SandboxImage is the image used by sandbox container. - SandboxImage string `toml:"sandbox_image" json:"sandboxImage,omitempty"` - // StatsCollectPeriod is the period (in seconds) of snapshots stats collection. - StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod,omitempty"` - // SystemdCgroup enables systemd cgroup support. - SystemdCgroup bool `toml:"systemd_cgroup" json:"systemdCgroup,omitempty"` - // EnableIPv6DAD enables IPv6 DAD. - // TODO(random-liu): Use optimistic_dad when it's GA. - EnableIPv6DAD bool `toml:"enable_ipv6_dad" json:"enableIPv6DAD,omitempty"` -} - -// Config contains toml config related cri-containerd daemon. -// TODO(random-liu): Make this an internal config object when we no longer support cri-containerd -// standalone mode. At that time, we can clean this up. -type Config struct { - // PluginConfig is the config for CRI plugin. - PluginConfig - // ContainerdRootDir is the root directory path for containerd. - ContainerdRootDir string `toml:"-" json:"containerdRootDir,omitempty"` - // ContainerdEndpoint is the containerd endpoint path. - ContainerdEndpoint string `toml:"-" json:"containerdEndpoint,omitempty"` - // SocketPath is the path to the socket which cri-containerd serves on. - // TODO(random-liu): Remove SocketPath when no longer support cri-containerd - // standalone mode. - SocketPath string `toml:"socket_path" json:"socketPath,omitempty"` - // RootDir is the root directory path for managing cri-containerd files - // (metadata checkpoint etc.) - RootDir string `toml:"root_dir" json:"rootDir,omitempty"` - // TODO(random-liu): Remove following fields when we no longer support cri-containerd - // standalone mode. - // CgroupPath is the path for the cgroup that cri-containerd is placed in. - CgroupPath string `toml:"cgroup_path" json:"cgroupPath,omitempty"` - // OOMScore adjust the cri-containerd's oom score - OOMScore int `toml:"oom_score" json:"oomScore,omitempty"` - // EnableProfiling is used for enable profiling via host:port/debug/pprof/ - EnableProfiling bool `toml:"profiling" json:"enableProfiling,omitempty"` - // ProfilingPort is the port for profiling via host:port/debug/pprof/ - ProfilingPort string `toml:"profiling_port" json:"profilingPort,omitempty"` - // ProfilingAddress is address for profiling via host:port/debug/pprof/ - ProfilingAddress string `toml:"profiling_addr" json:"profilingAddress,omitempty"` - // LogLevel is the logrus log level. - LogLevel string `toml:"log_level" json:"logLevel,omitempty"` -} - -// CRIContainerdOptions contains cri-containerd command line and toml options. -type CRIContainerdOptions struct { - // Config contains cri-containerd toml config - Config - // ConfigFilePath is the path to the TOML config file. - ConfigFilePath string `toml:"-"` -} - -// NewCRIContainerdOptions returns a reference to CRIContainerdOptions -func NewCRIContainerdOptions() *CRIContainerdOptions { - return &CRIContainerdOptions{} -} - -// AddFlags adds cri-containerd command line options to pflag. -func (c *CRIContainerdOptions) AddFlags(fs *pflag.FlagSet) { - defaults := DefaultConfig() - fs.StringVar(&c.ConfigFilePath, configFilePathArgName, - defaultConfigFilePath, "Path to the config file.") - fs.StringVar(&c.LogLevel, "log-level", - defaults.LogLevel, "Set the logging level [trace, debug, info, warn, error, fatal, panic].") - fs.StringVar(&c.SocketPath, "socket-path", - defaults.SocketPath, "Path to the socket which cri-containerd serves on.") - fs.StringVar(&c.RootDir, "root-dir", - defaults.RootDir, "Root directory path for cri-containerd managed files (metadata checkpoint etc).") - fs.StringVar(&c.ContainerdRootDir, "containerd-root-dir", - defaults.ContainerdRootDir, "Root directory path where containerd stores persistent data.") - fs.StringVar(&c.ContainerdEndpoint, "containerd-endpoint", - defaults.ContainerdEndpoint, "Path to the containerd endpoint.") - fs.StringVar(&c.ContainerdConfig.Snapshotter, "containerd-snapshotter", - defaults.ContainerdConfig.Snapshotter, "The snapshotter used by containerd.") - fs.StringVar(&c.ContainerdConfig.Runtime, "containerd-runtime", - defaults.ContainerdConfig.Runtime, "The runtime used by containerd.") - fs.StringVar(&c.ContainerdConfig.RuntimeEngine, "containerd-runtime-engine", - defaults.ContainerdConfig.RuntimeEngine, "Runtime engine used by containerd. Defaults to containerd's default if not specified.") - fs.StringVar(&c.ContainerdConfig.RuntimeRoot, "containerd-runtime-root", - defaults.ContainerdConfig.RuntimeRoot, "The directory used by containerd for runtime state. Defaults to containerd's default if not specified.") - fs.StringVar(&c.NetworkPluginBinDir, "network-bin-dir", - defaults.NetworkPluginBinDir, "The directory for putting network binaries.") - fs.StringVar(&c.NetworkPluginConfDir, "network-conf-dir", - defaults.NetworkPluginConfDir, "The directory for putting network plugin configuration files.") - fs.StringVar(&c.StreamServerAddress, "stream-addr", - defaults.StreamServerAddress, "The ip address streaming server is listening on. The default host interface is used if not specified.") - fs.StringVar(&c.StreamServerPort, "stream-port", - defaults.StreamServerPort, "The port streaming server is listening on.") - fs.StringVar(&c.CgroupPath, "cgroup-path", - defaults.CgroupPath, "The cgroup that cri-containerd is part of. Cri-containerd is not placed in a cgroup if none is specified.") - fs.BoolVar(&c.EnableSelinux, "enable-selinux", - defaults.EnableSelinux, "Enable selinux support. By default not enabled.") - fs.StringVar(&c.SandboxImage, "sandbox-image", - defaults.SandboxImage, "The image used by sandbox container.") - fs.IntVar(&c.StatsCollectPeriod, "stats-collect-period", - defaults.StatsCollectPeriod, "The period (in seconds) of snapshots stats collection.") - fs.BoolVar(&c.SystemdCgroup, "systemd-cgroup", - defaults.SystemdCgroup, "Enables systemd cgroup support. By default not enabled.") - fs.IntVar(&c.OOMScore, "oom-score", - defaults.OOMScore, "Adjust the cri-containerd's oom score.") - fs.BoolVar(&c.EnableProfiling, "profiling", - defaults.EnableProfiling, "Enable profiling via web interface host:port/debug/pprof/.") - fs.StringVar(&c.ProfilingPort, "profiling-port", - defaults.ProfilingPort, "Profiling port for web interface host:port/debug/pprof/.") - fs.StringVar(&c.ProfilingAddress, "profiling-addr", - defaults.ProfilingAddress, "Profiling address for web interface host:port/debug/pprof/.") - fs.BoolVar(&c.EnableIPv6DAD, "enable-ipv6-dad", - defaults.EnableIPv6DAD, "Enable IPv6 DAD (duplicate address detection) for pod sandbox network. Enabling this will increase pod sandbox start latency by several seconds.") - fs.Var(&c.Registry, "registry", - "Registry config for image pull eg --registry=myregistry.io=https://mymirror.io/ --registry=myregistry2.io=https://mymirror2.io/") -} - -// InitFlags load configurations from config file, and then overwrite with flags. -// This function must be called inside `Run`, at that time flags should have been -// parsed once. -// precedence: commandline > configfile > default -func (c *CRIContainerdOptions) InitFlags(fs *pflag.FlagSet) error { - // Load default config file if none provided - if _, err := toml.DecodeFile(c.ConfigFilePath, &c.Config); err != nil { - // the absence of default config file is normal case. - if !fs.Changed(configFilePathArgName) && os.IsNotExist(err) { - return nil - } - return err - } - // Add this for backward compatibility. - // TODO(random-liu): Remove this when we no longer support cri-containerd standalone mode. - if c.ContainerdConfig.RootDir != "" { - c.ContainerdRootDir = c.ContainerdConfig.RootDir - } - if c.ContainerdConfig.Endpoint != "" { - c.ContainerdEndpoint = c.ContainerdConfig.Endpoint - } - - // What is the reason for applying the command line twice? - // Because the values from command line have the highest priority. - // The path of toml configuration file if from the command line, - // and triggers the first parse. - // The first parse generates the default value and the value from command line at the same time. - // But the priority of the toml config value is higher than the default value, - // Without a way to insert the toml config value between the default value and the command line value. - // We parse twice one for default value, one for commandline value. - return fs.Parse(os.Args[1:]) -} - -// PrintDefaultTomlConfig print default toml config of cri-containerd. -func PrintDefaultTomlConfig() { - if err := toml.NewEncoder(os.Stdout).Encode(DefaultConfig()); err != nil { - fmt.Println(err) - return - } -} - -// DefaultConfig returns default configurations of cri-containerd. -func DefaultConfig() Config { - return Config{ - PluginConfig: PluginConfig{ - CniConfig: CniConfig{ - NetworkPluginBinDir: "/opt/cni/bin", - NetworkPluginConfDir: "/etc/cni/net.d", - }, - ContainerdConfig: ContainerdConfig{ - Snapshotter: containerd.DefaultSnapshotter, - Runtime: "io.containerd.runtime.v1.linux", - RuntimeEngine: "", - RuntimeRoot: "", - }, - StreamServerAddress: "", - StreamServerPort: "10010", - EnableSelinux: false, - SandboxImage: "gcr.io/google_containers/pause:3.0", - StatsCollectPeriod: 10, - SystemdCgroup: false, - EnableIPv6DAD: false, - Registry: Registry{ - Mirrors: map[string]Mirror{ - "docker.io": { - Endpoints: []string{"https://registry-1.docker.io"}, - }, - }, - }, - }, - ContainerdRootDir: "/var/lib/containerd", - ContainerdEndpoint: "/run/containerd/containerd.sock", - SocketPath: "/var/run/cri-containerd.sock", - RootDir: "/var/lib/cri-containerd", - CgroupPath: "", - OOMScore: -999, - EnableProfiling: true, - ProfilingPort: "10011", - ProfilingAddress: "127.0.0.1", - LogLevel: "info", - } -} diff --git a/cmd/cri-containerd/options/registry.go b/cmd/cri-containerd/options/registry.go deleted file mode 100644 index 7f057ee4c4f1..000000000000 --- a/cmd/cri-containerd/options/registry.go +++ /dev/null @@ -1,88 +0,0 @@ -/* -Copyright 2018 The Containerd Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package options - -import ( - "fmt" - "net/url" - "strings" -) - -// Mirror contains the config related to the registry mirror -type Mirror struct { - Endpoints []string `toml:"endpoint" json:"endpoint,omitempty"` - // TODO (Abhi) We might need to add auth per namespace. Looks like - // image auth information is passed by kube itself. -} - -// Registry is registry settings configured -type Registry struct { - Mirrors map[string]Mirror `toml:"mirrors" json:"mirrors,omitempty"` -} - -// String returns the string format of registry type -func (r *Registry) String() string { - // Its not used hence return empty string - return "" -} - -// Set validates and converts into the internal registry struct -func (r *Registry) Set(s string) error { - // --registry docker.io=https://mymirror.io,http://mymirror2.io - // If no option is set then return format error - if len(s) == 0 { - return fmt.Errorf("incomplete registry mirror option") - } - var mirrors []string - host := "docker.io" - opt := strings.Split(s, "=") - if len(opt) > 1 { - // If option is set in the format "mynamespace.io=https://mymirror.io,https://mymirror2.io" - // Then associate the mirror urls for the namespace only" - host = opt[0] - mirrors = strings.Split(opt[1], ",") - } else { - // If option is set in the format "https://mymirror.io,https://mymirror.io" - // Then associate mirror against default docker.io namespace - mirrors = strings.Split(opt[0], ",") - } - - // Validate the format of the urls passed - for _, u := range mirrors { - _, err := url.Parse(u) - if err != nil { - return fmt.Errorf("invalid registry mirror url format %v: %v", u, err) - } - } - - if r.Mirrors == nil { - r.Mirrors = make(map[string]Mirror) - } - if _, ok := r.Mirrors[host]; !ok { - r.Mirrors[host] = Mirror{} - } - m := r.Mirrors[host] - m.Endpoints = append(m.Endpoints, mirrors...) - r.Mirrors[host] = m - - return nil -} - -// Type returns a string name for the option type -func (r *Registry) Type() string { - return "list" -} diff --git a/cmd/ctrcri/ctrcri.go b/cmd/ctrcri/ctrcri.go index 63276cf75368..67811d4c38ad 100644 --- a/cmd/ctrcri/ctrcri.go +++ b/cmd/ctrcri/ctrcri.go @@ -20,10 +20,10 @@ import ( "os" "time" + "github.com/containerd/containerd/defaults" "github.com/spf13/cobra" "github.com/spf13/pflag" - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" "github.com/containerd/cri-containerd/pkg/version" ) @@ -56,9 +56,7 @@ var ( ) func addGlobalFlags(fs *pflag.FlagSet) { - // TODO(random-liu): Change default to containerd/defaults.DefaultAddress after cri plugin - // become default. - fs.StringVar(&address, "address", options.DefaultConfig().SocketPath, "address for containerd's GRPC server.") + fs.StringVar(&address, "address", defaults.DefaultAddress, "address for containerd's GRPC server.") fs.DurationVar(&timeout, "timeout", defaultTimeout, "timeout for containerd grpc connection.") } diff --git a/contrib/ansible/README.md b/contrib/ansible/README.md index 6c049d578b21..a75a8759d1e7 100644 --- a/contrib/ansible/README.md +++ b/contrib/ansible/README.md @@ -1,4 +1,4 @@ -# Kubernetes Cluster with Containerd and CRI-Containerd +# Kubernetes Cluster with Containerd

diff --git a/contrib/ansible/cri-containerd.yaml b/contrib/ansible/cri-containerd.yaml index 52aa1a8b56b1..3eb70c6f6888 100644 --- a/contrib/ansible/cri-containerd.yaml +++ b/contrib/ansible/cri-containerd.yaml @@ -8,7 +8,7 @@ - include_tasks: tasks/bootstrap_centos.yaml # Contains tasks bootstrap components for centos systems when: ansible_distribution == "CentOS" - include_tasks: tasks/k8s.yaml # Contains tasks kubernetes component installation - - include_tasks: tasks/binaries.yaml # Contains tasks for pulling containerd and cri-containerd components + - include_tasks: tasks/binaries.yaml # Contains tasks for pulling containerd components - name: "Create a directory for containerd config" file: path=/etc/containerd state=directory @@ -21,22 +21,9 @@ [cgroup] path = "/runtime" - - name: "Create a directory for cri-containerd config" - file: path=/etc/cri-containerd state=directory - - - name: "Add cri-containerd config file" - blockinfile: - path: /etc/cri-containerd/config.toml - create: yes - block: | - cgroup_path = "/runtime" - - name: "Start Containerd" systemd: name=containerd daemon_reload=yes state=started enabled=yes - - name: "Start CRI-Containerd" - systemd: name=cri-containerd daemon_reload=yes state=started enabled=yes - - name: "Load br_netfilter kernel module" modprobe: name: br_netfilter @@ -59,7 +46,7 @@ - name: "Add runtime args in kubelet conf" lineinfile: dest: "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" - line: "Environment=\"KUBELET_EXTRA_ARGS= --runtime-cgroups=/runtime --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=/var/run/cri-containerd.sock\"" + line: "Environment=\"KUBELET_EXTRA_ARGS= --runtime-cgroups=/runtime --container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=/run/containerd/containerd.sock\"" insertafter: '\[Service\]' when: check_args.stdout == "" @@ -70,5 +57,5 @@ - name: "Pre-pull pause container image" shell: | /usr/local/bin/ctr pull gcr.io/google_containers/pause:3.0 - /usr/local/bin/crictl --runtime-endpoint /var/run/cri-containerd.sock \ + /usr/local/bin/crictl --runtime-endpoint /run/containerd/containerd.sock \ pull gcr.io/google_containers/pause:3.0 diff --git a/contrib/ansible/tasks/binaries.yaml b/contrib/ansible/tasks/binaries.yaml index 2bcc444955ed..808c7a72acb4 100644 --- a/contrib/ansible/tasks/binaries.yaml +++ b/contrib/ansible/tasks/binaries.yaml @@ -1,5 +1,5 @@ --- -- name: "Get Containerd and CRI-Containerd" +- name: "Get Containerd" unarchive: src: "https://storage.googleapis.com/cri-containerd-release/cri-containerd-{{ cri_containerd_release_version }}.linux-amd64.tar.gz" dest: "/" diff --git a/contrib/ansible/vars/vars.yaml b/contrib/ansible/vars/vars.yaml index 8cbe743bd181..d9c828a4cbc1 100644 --- a/contrib/ansible/vars/vars.yaml +++ b/contrib/ansible/vars/vars.yaml @@ -1,7 +1,4 @@ --- cri_containerd_release_version: 1.0.0-beta.0 -cri_release_directory: /opt/cri-containerd/ -local_bin_dir: /usr/local/bin/ -local_sbin_dir: /usr/local/sbin/ cni_bin_dir: /opt/cni/bin/ cni_conf_dir: /etc/cni/net.d/ diff --git a/contrib/systemd-units/cri-containerd.service b/contrib/systemd-units/cri-containerd.service deleted file mode 100644 index b82e85e27b5d..000000000000 --- a/contrib/systemd-units/cri-containerd.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Kubernetes containerd CRI shim -Requires=network-online.target -After=containerd.service - -[Service] -Restart=always -RestartSec=5 -LimitNOFILE=1048576 -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNPROC=infinity -LimitCORE=infinity -ExecStart=/usr/local/bin/cri-containerd - -[Install] -WantedBy=multi-user.target diff --git a/cri.go b/cri.go index fdcdf81cb4c8..4da21e91078a 100644 --- a/cri.go +++ b/cri.go @@ -17,6 +17,7 @@ limitations under the License. package cri import ( + "flag" "path/filepath" "github.com/containerd/containerd/log" @@ -24,8 +25,9 @@ import ( "github.com/containerd/containerd/plugin" imagespec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/sirupsen/logrus" - "github.com/containerd/cri-containerd/cmd/cri-containerd/options" + criconfig "github.com/containerd/cri-containerd/pkg/config" "github.com/containerd/cri-containerd/pkg/server" ) @@ -35,7 +37,7 @@ const criVersion = "v1alpha2" // TODO(random-liu): Use github.com/pkg/errors for our errors. // Register CRI service plugin func init() { - config := options.DefaultConfig().PluginConfig + config := criconfig.DefaultConfig() plugin.Register(&plugin.Registration{ Type: plugin.GRPCPlugin, ID: "cri", @@ -57,8 +59,8 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { ic.Meta.Platforms = []imagespec.Platform{platforms.DefaultSpec()} ic.Meta.Exports = map[string]string{"CRIVersion": criVersion} ctx := ic.Context - pluginConfig := ic.Config.(*options.PluginConfig) - c := options.Config{ + pluginConfig := ic.Config.(*criconfig.PluginConfig) + c := criconfig.Config{ PluginConfig: *pluginConfig, // This is a hack. We assume that containerd root directory // is one level above plugin directory. @@ -69,6 +71,10 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { } log.G(ctx).Infof("Start cri plugin with config %+v", c) + if err := setGLogLevel(); err != nil { + return nil, errors.Wrap(err, "failed to set glog level") + } + s, err := server.NewCRIContainerdService(c) if err != nil { return nil, errors.Wrap(err, "failed to create CRI service") @@ -77,10 +83,32 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) { // Use a goroutine to initialize cri service. The reason is that currently // cri service requires containerd to be initialize. go func() { - if err := s.Run(false); err != nil { + if err := s.Run(); err != nil { log.G(ctx).WithError(err).Fatal("Failed to run CRI service") } // TODO(random-liu): Whether and how we can stop containerd. }() return s, nil } + +// Set glog level. +func setGLogLevel() error { + l := logrus.GetLevel() + if err := flag.Set("logtostderr", "true"); err != nil { + return err + } + switch l { + case log.TraceLevel: + return flag.Set("v", "5") + case logrus.DebugLevel: + return flag.Set("v", "4") + case logrus.InfoLevel: + return flag.Set("v", "2") + // glog doesn't support following filters. Defaults to v=0. + case logrus.WarnLevel: + case logrus.ErrorLevel: + case logrus.FatalLevel: + case logrus.PanicLevel: + } + return nil +} diff --git a/hack/install-deps.sh b/hack/install-deps.sh index 803d9570141e..76a73665f3e1 100755 --- a/hack/install-deps.sh +++ b/hack/install-deps.sh @@ -133,8 +133,7 @@ fi # Install containerd checkout_repo ${CONTAINERD_PKG} ${CONTAINERD_VERSION} ${CONTAINERD_REPO} cd ${GOPATH}/src/${CONTAINERD_PKG} -# Build no_cri version and run standalone cri-containerd. -make BUILDTAGS="${BUILDTAGS} no_cri" +make BUILDTAGS="${BUILDTAGS}" # containerd make install requires `go` to work. Explicitly # set PATH to make sure it can find `go` even with `sudo`. ${sudo} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}" @@ -146,7 +145,7 @@ make crictl ${sudo} make install-crictl -e BINDIR=${CRICTL_DIR} GOPATH=${GOPATH} ${sudo} mkdir -p ${CRICTL_CONFIG_DIR} ${sudo} bash -c 'cat >'${CRICTL_CONFIG_DIR}'/crictl.yaml < \ - /etc/profile.d/cri-containerd_env.sh +echo "export PATH=${CONTAINERD_HOME}/usr/local/bin/:${CONTAINERD_HOME}/usr/local/sbin/:\$PATH" > \ + /etc/profile.d/containerd_env.sh # EXTRA_INIT_SCRIPT is the name of the extra init script after being downloaded. EXTRA_INIT_SCRIPT="extra-init.sh" diff --git a/test/e2e_node/benchmark-config.yaml b/test/e2e_node/benchmark-config.yaml index 8c7e2d2b8c61..7541bd7ae3ed 100644 --- a/test/e2e_node/benchmark-config.yaml +++ b/test/e2e_node/benchmark-config.yaml @@ -4,21 +4,21 @@ images: image: cos-stable-60-9592-90-0 project: cos-cloud machine: n1-standard-1 - metadata: "user-data