Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Automated cherry pick of #41944 #43403

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
22 changes: 15 additions & 7 deletions cluster/gce/gci/configure-helper.sh
Original file line number Diff line number Diff line change
Expand Up @@ -547,7 +547,7 @@ function start-kubelet {
flags+=" --cluster-dns=${DNS_SERVER_IP}"
flags+=" --cluster-domain=${DNS_DOMAIN}"
flags+=" --config=/etc/kubernetes/manifests"
flags+=" --experimental-mounter-path=${KUBE_HOME}/bin/mounter"
flags+=" --experimental-mounter-path=${CONTAINERIZED_MOUNTER_HOME}/mounter"
flags+=" --experimental-check-node-capabilities-before-mount=true"

if [[ -n "${KUBELET_PORT:-}" ]]; then
Expand Down Expand Up @@ -782,6 +782,18 @@ function compute-master-manifest-variables {
fi
}

# A helper function that bind mounts kubelet dirs for running mount in a chroot
function prepare-mounter-rootfs {
echo "Prepare containerized mounter"
mount --bind "${CONTAINERIZED_MOUNTER_HOME}" "${CONTAINERIZED_MOUNTER_HOME}"
mount -o remount,exec "${CONTAINERIZED_MOUNTER_HOME}"
CONTAINERIZED_MOUNTER_ROOTFS="${CONTAINERIZED_MOUNTER_HOME}/rootfs"
mount --rbind /var/lib/kubelet/ "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
mount --make-rshared "${CONTAINERIZED_MOUNTER_ROOTFS}/var/lib/kubelet"
mount --bind -o ro /proc "${CONTAINERIZED_MOUNTER_ROOTFS}/proc"
mount --bind -o ro /dev "${CONTAINERIZED_MOUNTER_ROOTFS}/dev"
}

# A helper function for removing salt configuration and comments from a file.
# This is mainly for preparing a manifest file.
#
Expand Down Expand Up @@ -1283,15 +1295,11 @@ function override-kubectl {
echo "export PATH=${KUBE_HOME}/bin:\$PATH" > /etc/profile.d/kube_env.sh
}

function pre-warm-mounter {
echo "prewarming mounter"
${KUBE_HOME}/bin/mounter &> /dev/null
}

########### Main Function ###########
echo "Start to configure instance for kubernetes"

KUBE_HOME="/home/kubernetes"
CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
exit 1
Expand Down Expand Up @@ -1327,7 +1335,6 @@ fi

override-kubectl
# Run the containerized mounter once to pre-cache the container image.
pre-warm-mounter
assemble-docker-flags
load-docker-images
start-kubelet
Expand Down Expand Up @@ -1355,4 +1362,5 @@ else
fi
start-fluentd
reset-motd
prepare-mounter-rootfs
echo "Done for the configuration for kubernetes"
26 changes: 12 additions & 14 deletions cluster/gce/gci/configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -99,19 +99,18 @@ function split-commas {
}

function install-gci-mounter-tools {
local -r rkt_version="v1.18.0"
local -r gci_mounter_version="v2"
local -r rkt_binary_sha1="75fc8f29c79bc9e505f3e7f6e8fadf2425c21967"
local -r rkt_stage1_fly_sha1="474df5a1f934960ba669b360ab713d0a54283091"
local -r gci_mounter_sha1="851e841d8640d6a05e64e22c493f5ac3c4cba561"
download-or-bust "${rkt_binary_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/rkt"
download-or-bust "${rkt_stage1_fly_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/stage1-fly.aci"
download-or-bust "${gci_mounter_sha1}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-${gci_mounter_version}.aci"
local -r rkt_dst="${KUBE_HOME}/bin/"
mv "${KUBE_HOME}/rkt" "${rkt_dst}/rkt"
mv "${KUBE_HOME}/stage1-fly.aci" "${rkt_dst}/stage1-fly.aci"
mv "${KUBE_HOME}/gci-mounter-${gci_mounter_version}.aci" "${rkt_dst}/gci-mounter-${gci_mounter_version}.aci"
chmod a+x "${rkt_dst}/rkt"
CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
mkdir "${CONTAINERIZED_MOUNTER_HOME}"
chmod a+x "${CONTAINERIZED_MOUNTER_HOME}"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have we tested this with u+x to see if that would work too? I saw this was noted in #41944 as well, so it may have been decided offline that a+x is ok, but it would be nice to not have to be so broad with the permissions.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

u+x should work, but maybe we could use a different PR to change it on both 1.5 and 1.6?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A different PR is fine.

mkdir "${CONTAINERIZED_MOUNTER_HOME}/rootfs"
local -r mounter_tar_sha="8003b798cf33c7f91320cd6ee5cec4fa22244571"
download-or-bust "${mounter_tar_sha}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar"
cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${CONTAINERIZED_MOUNTER_HOME}/mounter"
chmod a+x "${CONTAINERIZED_MOUNTER_HOME}/mounter"
mv "${KUBE_HOME}/mounter.tar" /tmp/mounter.tar
tar xvf /tmp/mounter.tar -C "${CONTAINERIZED_MOUNTER_HOME}/rootfs"
rm /tmp/mounter.tar
mkdir "${CONTAINERIZED_MOUNTER_HOME}/rootfs/var/lib/kubelet"
}

# Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
Expand Down Expand Up @@ -187,7 +186,6 @@ function install-kube-binary-config {
xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
fi
cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter"
cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh"
chmod -R 755 "${kube_bin}"

Expand Down
1 change: 0 additions & 1 deletion cluster/gce/gci/master.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ write_files:
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
ExecStart=/home/kubernetes/bin/configure-helper.sh

[Install]
Expand Down
Binary file modified cluster/gce/gci/mounter/mounter
Binary file not shown.
93 changes: 93 additions & 0 deletions cluster/gce/gci/mounter/mounter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
)

const (
// Location of the mount file to use
chrootCmd = "chroot"
mountCmd = "mount"
rootfs = "rootfs"
nfsRPCBindErrMsg = "mount.nfs: rpc.statd is not running but is required for remote locking.\nmount.nfs: Either use '-o nolock' to keep locks local, or start statd.\nmount.nfs: an incorrect mount option was specified\n"
rpcBindCmd = "/sbin/rpcbind"
defaultRootfs = "/home/kubernetes/containerized_mounter/rootfs"
)

func main() {

if len(os.Args) < 2 {
fmt.Fprintf(os.Stderr, "Command failed: must provide a command to run.\n")
return
}
path, _ := filepath.Split(os.Args[0])
rootfsPath := filepath.Join(path, rootfs)
if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
rootfsPath = defaultRootfs
}
command := os.Args[1]
switch command {
case mountCmd:
mountErr := mountInChroot(rootfsPath, os.Args[2:])
if mountErr != nil {
fmt.Fprintf(os.Stderr, "Mount failed: %v", mountErr)
os.Exit(1)
}
default:
fmt.Fprintf(os.Stderr, "Unknown command, must be %s", mountCmd)
os.Exit(1)

}
}

// MountInChroot is to run mount within chroot with the passing root directory
func mountInChroot(rootfsPath string, args []string) error {
if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
return fmt.Errorf("Path <%s> does not exist.\n", rootfsPath)
}
args = append([]string{rootfsPath, mountCmd}, args...)
output, err := exec.Command(chrootCmd, args...).CombinedOutput()
if err == nil {
return err
}

if !strings.EqualFold(string(output), nfsRPCBindErrMsg) {
// Mount failed but not because of RPC bind error
return fmt.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s\n", err, chrootCmd, args, string(output))
}

// Mount failed because it is NFS V3 and we need to run rpcBind
output, err = exec.Command(chrootCmd, rootfsPath, rpcBindCmd, "-w").CombinedOutput()
if err != nil {
return fmt.Errorf("Mount issued for NFS V3 but unable to run rpcbind:\n Output: %s\n Error: %v", string(output), err)
}

// Rpcbind is running, try mounting again
output, err = exec.Command(chrootCmd, args...).CombinedOutput()

if err != nil {
return fmt.Errorf("Mount failed for NFS V3 even after running rpcBind %s, %v", string(output), err)
}

return nil
}
1 change: 0 additions & 1 deletion cluster/gce/gci/node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ write_files:
Type=oneshot
RemainAfterExit=yes
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
ExecStart=/home/kubernetes/bin/configure-helper.sh

[Install]
Expand Down
1 change: 0 additions & 1 deletion cluster/gce/trusty/configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,6 @@ install_kube_binary_config() {
# This should be the case of GCI.
readonly kube_bin="${kube_home}/bin"
mkdir -p "${kube_bin}"
mount --bind "${kube_bin}" "${kube_bin}"
mount -o remount,rw,exec "${kube_bin}"
cp "${src_dir}/kubelet" "${kube_bin}"
cp "${src_dir}/kubectl" "${kube_bin}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ spec:
command:
- /bin/sh
- -c
- "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.7 gcr.io/google_containers/mounttest-user:0.3 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.0 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.6 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;"
- "for i in gcr.io/google_containers/busybox gcr.io/google_containers/busybox:1.24 gcr.io/google_containers/dnsutils:e2e gcr.io/google_containers/eptest:0.1 gcr.io/google_containers/fakegitserver:0.1 gcr.io/google_containers/hostexec:1.2 gcr.io/google_containers/iperf:e2e gcr.io/google_containers/jessie-dnsutils:e2e gcr.io/google_containers/liveness:e2e gcr.io/google_containers/mounttest:0.8 gcr.io/google_containers/mounttest-user:0.5 gcr.io/google_containers/netexec:1.4 gcr.io/google_containers/netexec:1.7 gcr.io/google_containers/nettest:1.7 gcr.io/google_containers/nettest:1.8 gcr.io/google_containers/nginx-slim:0.7 gcr.io/google_containers/nginx-slim:0.8 gcr.io/google_containers/n-way-http:1.0 gcr.io/google_containers/pause:2.0 gcr.io/google_containers/pause-amd64:3.0 gcr.io/google_containers/porter:cd5cb5791ebaa8641955f0e8c2a9bed669b1eaab gcr.io/google_containers/portforwardtester:1.2 gcr.io/google_containers/redis:e2e gcr.io/google_containers/resource_consumer:beta4 gcr.io/google_containers/resource_consumer/controller:beta4 gcr.io/google_containers/serve_hostname:v1.4 gcr.io/google_containers/test-webserver:e2e gcr.io/google_containers/ubuntu:14.04 gcr.io/google_containers/update-demo:kitten gcr.io/google_containers/update-demo:nautilus gcr.io/google_containers/volume-ceph:0.1 gcr.io/google_containers/volume-gluster:0.2 gcr.io/google_containers/volume-iscsi:0.1 gcr.io/google_containers/volume-nfs:0.8 gcr.io/google_containers/volume-rbd:0.1 gcr.io/google_samples/gb-redisslave:v1 gcr.io/google_containers/redis:v1; do echo $(date '+%X') pulling $i; docker pull $i 1>/dev/null; done; exit 0;"
securityContext:
privileged: true
volumeMounts:
Expand Down
1 change: 1 addition & 0 deletions hack/.linted_packages
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
cluster/addons/fluentd-elasticsearch/es-image
cluster/gce/gci/mounter
cluster/images/etcd/attachlease
cluster/images/etcd/rollback
cmd/clicheck
Expand Down
5 changes: 0 additions & 5 deletions pkg/util/mount/mount.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,6 @@ func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string,
// It provides options to override the default mounter behavior.
// mounterPath allows using an alternative to `/bin/mount` for mounting.
func New(mounterPath string) Interface {
// If mounter-path flag is not set, use default mount path
if mounterPath == "" {
mounterPath = defaultMountCommand
}

return &Mounter{
mounterPath: mounterPath,
}
Expand Down
21 changes: 12 additions & 9 deletions pkg/util/mount/mount_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,23 +63,23 @@ type Mounter struct {
// currently come from mount(8), e.g. "ro", "remount", "bind", etc. If no more option is
// required, call Mount with an empty string list or nil.
func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
// Path to mounter binary. Set to mount accessible via $PATH by default.
// Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty.
// All Linux distros are expected to be shipped with a mount utility that an support bind mounts.
mounterPath := defaultMountCommand
mounterPath := ""
bind, bindRemountOpts := isBind(options)
if bind {
err := doMount(mounterPath, source, target, fstype, []string{"bind"})
err := doMount(mounterPath, defaultMountCommand, source, target, fstype, []string{"bind"})
if err != nil {
return err
}
return doMount(mounterPath, source, target, fstype, bindRemountOpts)
return doMount(mounterPath, defaultMountCommand, source, target, fstype, bindRemountOpts)
}
// The list of filesystems that require containerized mounter on GCI image cluster
fsTypesNeedMounter := sets.NewString("nfs", "glusterfs")
fsTypesNeedMounter := sets.NewString("nfs", "glusterfs", "ceph", "cifs")
if fsTypesNeedMounter.Has(fstype) {
mounterPath = mounter.mounterPath
}
return doMount(mounterPath, source, target, fstype, options)
return doMount(mounterPath, defaultMountCommand, source, target, fstype, options)
}

// isBind detects whether a bind mount is being requested and makes the remount options to
Expand Down Expand Up @@ -107,10 +107,13 @@ func isBind(options []string) (bool, []string) {
return bind, bindRemountOpts
}

// doMount runs the mount command.
func doMount(mountCmd string, source string, target string, fstype string, options []string) error {
glog.V(4).Infof("Mounting %s %s %s %v with command: %q", source, target, fstype, options, mountCmd)
// doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used.
func doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string) error {
mountArgs := makeMountArgs(source, target, fstype, options)
if len(mounterPath) > 0 {
mountArgs = append([]string{mountCmd}, mountArgs...)
mountCmd = mounterPath
}

glog.V(4).Infof("Mounting cmd (%s) with arguments (%s)", mountCmd, mountArgs)
command := exec.Command(mountCmd, mountArgs...)
Expand Down
34 changes: 32 additions & 2 deletions test/e2e/common/volumes.go
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,36 @@ var _ = framework.KubeDescribe("GCP Volumes", func() {
})
})

framework.KubeDescribe("NFSv3", func() {
It("should be mountable for NFSv3 [Volume]", func() {
config := VolumeTestConfig{
namespace: namespace.Name,
prefix: "nfs",
serverImage: "gcr.io/google_containers/volume-nfs:0.8",
serverPorts: []int{2049},
}

defer func() {
if clean {
volumeTestCleanup(f, config)
}
}()
pod := startVolumeServer(f, config)
serverIP := pod.Status.PodIP
framework.Logf("NFS server IP address: %v", serverIP)

volume := api.VolumeSource{
NFS: &api.NFSVolumeSource{
Server: serverIP,
Path: "/exports",
ReadOnly: true,
},
}
// Must match content of test/images/volume-tester/nfs/index.html
testVolumeClient(f, config, volume, nil, "Hello from NFS!")
})
})

////////////////////////////////////////////////////////////////////////
// Gluster
////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -488,10 +518,10 @@ func isTestEnabled(c clientset.Interface) bool {
}

// For cluster e2e test, because nodeName is empty, retrieve the node objects from api server
// and check their images. Only run NFSv4 and GlusterFS if nodes are using GCI image for now.
// and check their images. Only run NFS and GlusterFS tests if nodes are using GCI image for now.
nodes := framework.GetReadySchedulableNodesOrDie(c)
for _, node := range nodes.Items {
if !strings.Contains(node.Status.NodeInfo.OSImage, "Google Container-VM") {
if !strings.Contains(node.Status.NodeInfo.OSImage, "Container-Optimized OS") {
return false
}
}
Expand Down
19 changes: 12 additions & 7 deletions test/e2e_node/jenkins/gci-init.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,15 @@
runcmd:
- mount /tmp /tmp -o remount,exec,suid
- usermod -a -G docker jenkins
- mkdir -p /home/kubernetes/bin/
- mount -B /home/kubernetes/bin /home/kubernetes/bin
- mount -B -o remount,exec /home/kubernetes/bin
- wget https://storage.googleapis.com/kubernetes-release/rkt/v1.18.0/rkt -O /home/kubernetes/bin/rkt
- wget https://storage.googleapis.com/kubernetes-release/rkt/v1.18.0/stage1-fly.aci -O /home/kubernetes/bin/stage1-fly.aci
- wget https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-v2.aci -O /home/kubernetes/bin/gci-mounter-v2.aci
- chmod a+x /home/kubernetes/bin/rkt
- mkdir -p /var/lib/kubelet
- mkdir -p /home/kubernetes/containerized_mounter/rootfs
- mount --bind /home/kubernetes/containerized_mounter/ /home/kubernetes/containerized_mounter/
- mount -o remount, exec /home/kubernetes/containerized_mounter/
- wget https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar -O /tmp/mounter.tar
- tar xvf /tmp/mounter.tar -C /home/kubernetes/containerized_mounter/rootfs
- mkdir -p /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
- mount --rbind /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
- mount --make-rshared /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet
- mount --bind /proc /home/kubernetes/containerized_mounter/rootfs/proc
- mount --bind /dev /home/kubernetes/containerized_mounter/rootfs/dev
- rm /tmp/mounter.tar