kubernetes · jessfraz · Dec 1, 2016 · Nov 23, 2016
diff --git a/build/common.sh b/build/common.sh
@@ -946,6 +946,7 @@ function kube::release::package_kube_manifests_tarball() {
   cp "${salt_dir}/e2e-image-puller/e2e-image-puller.manifest" "${dst_dir}/"
   cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}/trusty-configure-helper.sh"
   cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh"
+  cp "${KUBE_ROOT}/cluster/gce/gci/mounter/mounter" "${dst_dir}/gci-mounter"
   cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"
   cp -r "${salt_dir}/kube-admission-controls/limit-range" "${dst_dir}"
   local objects

diff --git a/cluster/gce/gci/configure-helper.sh b/cluster/gce/gci/configure-helper.sh
@@ -473,6 +473,7 @@ function start-kubelet {
   flags+=" --cluster-dns=${DNS_SERVER_IP}"
   flags+=" --cluster-domain=${DNS_DOMAIN}"
   flags+=" --config=/etc/kubernetes/manifests"
+  flags+=" --experimental-mounter-path=${KUBE_HOME}/bin/mounter"
 
   if [[ -n "${KUBELET_PORT:-}" ]]; then
     flags+=" --port=${KUBELET_PORT}"
@@ -1173,6 +1174,11 @@ For Kubernetes copyright and licensing information, see:
 EOF
 }
 
+function pre-warm-mounter {
+    echo "prewarming mounter"
+    ${KUBE_HOME}/bin/mounter &> /dev/null
+}
+
 function override-kubectl {
     echo "export PATH=${KUBE_HOME}/bin:\$PATH" > /etc/profile.d/kube_env.sh
 }
@@ -1211,6 +1217,9 @@ else
 fi
 
 override-kubectl
+
+# Run the containerized mounter once to pre-cache the container image.
+pre-warm-mounter
 assemble-docker-flags
 load-docker-images
 start-kubelet

diff --git a/cluster/gce/gci/configure.sh b/cluster/gce/gci/configure.sh
@@ -98,13 +98,20 @@ function split-commas {
   echo $1 | tr "," "\n"
 }
 
-function install-rkt {
-    local -r rkt_binary="rkt-v1.18.0"
-    local -r rkt_sha1="75fc8f29c79bc9e505f3e7f6e8fadf2425c21967"
-    download-or-bust "${rkt_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_binary}"
-    local -r rkt_dst="${KUBE_HOME}/bin/rkt"
-    mv "${KUBE_HOME}/${rkt_binary}" "${rkt_dst}"
-    chmod a+x "${rkt_dst}"
+function install-gci-mounter-tools {
+    local -r rkt_version="v1.18.0"
+    local -r gci_mounter_version="v2"
+    local -r rkt_binary_sha1="75fc8f29c79bc9e505f3e7f6e8fadf2425c21967"
+    local -r rkt_stage1_fly_sha1="474df5a1f934960ba669b360ab713d0a54283091"
+    local -r gci_mounter_sha1="851e841d8640d6a05e64e22c493f5ac3c4cba561"
+    download-or-bust "${rkt_binary_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/rkt"
+    download-or-bust "${rkt_stage1_fly_sha1}" "https://storage.googleapis.com/kubernetes-release/rkt/${rkt_version}/stage1-fly.aci"
+    download-or-bust "${gci_mounter_sha1}" "https://storage.googleapis.com/kubernetes-release/gci-mounter/gci-mounter-${gci_mounter_version}.aci"
+    local -r rkt_dst="${KUBE_HOME}/bin/"
+    mv "${KUBE_HOME}/rkt" "${rkt_dst}/rkt"
+    mv "${KUBE_HOME}/stage1-fly.aci" "${rkt_dst}/stage1-fly.aci"
+    mv "${KUBE_HOME}/gci-mounter-${gci_mounter_version}.aci" "${rkt_dst}/gci-mounter-${gci_mounter_version}.aci"
+    chmod a+x "${rkt_dst}/rkt"
 }
 
 # Downloads kubernetes binaries and kube-system manifest tarball, unpacks them,
@@ -180,11 +187,12 @@ function install-kube-binary-config {
       xargs sed -ri "s@(image\":\s+\")gcr.io/google_containers@\1${kube_addon_registry}@"
   fi
   cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_HOME}/bin/configure-helper.sh"
+  cp "${dst_dir}/kubernetes/gci-trusty/gci-mounter" "${KUBE_HOME}/bin/mounter"
   cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_HOME}/bin/health-monitor.sh"
   chmod -R 755 "${kube_bin}"
 
-  # Install rkt binary to allow mounting storage volumes in GCI
-  install-rkt
+  # Install gci mounter related artifacts to allow mounting storage volumes in GCI
+  install-gci-mounter-tools
 
   # Clean up.
   rm -rf "${KUBE_HOME}/kubernetes"

diff --git a/cluster/gce/gci/master.yaml b/cluster/gce/gci/master.yaml
@@ -34,6 +34,7 @@ write_files:
       Type=oneshot
       RemainAfterExit=yes
       ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
       ExecStart=/home/kubernetes/bin/configure-helper.sh
 
       [Install]

diff --git a/cluster/gce/gci/mounter/Dockerfile b/cluster/gce/gci/mounter/Dockerfile
@@ -0,0 +1,20 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:xenial
+MAINTAINER vishh@google.com
+
+RUN apt-get update && apt-get install -y netbase nfs-common=1:1.2.8-9ubuntu12 glusterfs-client=3.7.6-1ubuntu1
+
+ENTRYPOINT ["/bin/mount"]
diff --git a/cluster/gce/gci/mounter/Makefile b/cluster/gce/gci/mounter/Makefile
@@ -0,0 +1,30 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TAG=v2
+REGISTRY=gcr.io/google_containers
+IMAGE=gci-mounter
+
+all: container
+
+container:
+	docker build --pull -t ${REGISTRY}/${IMAGE}:${TAG} .
+
+push: 
+	gcloud docker -- push ${REGISTRY}/${IMAGE}:${TAG}
+
+upload:
+	./stage-upload.sh ${TAG} ${REGISTRY}/${IMAGE}:${TAG}
+
+.PHONY: all container push
diff --git a/cluster/gce/gci/mounter/mounter b/cluster/gce/gci/mounter/mounter
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Copyright 2014 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+MOUNTER_VERSION=v2
+MOUNTER_USER=root
+ROOT_DIR=/home/kubernetes/bin
+RKT_BINARY=${ROOT_DIR}/rkt
+STAGE1_ACI=${ROOT_DIR}/stage1-fly.aci
+MOUNTER_ACI=${ROOT_DIR}/gci-mounter-${MOUNTER_VERSION}.aci
+MOUNTER_IMAGE=gcr.io/google_containers/gci-mounter:${MOUNTER_VERSION}
+
+function gc {
+    # Attempt to garbage collect rkt pods with 5 retries.
+    # Rkt pods end up creating new copies of mounts on the host. Hence it is ideal to clean them up right away.
+    attempt=0
+    until [ $attempt -ge 5 ]; do
+	${RKT_BINARY} gc --grace-period=0s &> /dev/null && break
+	attempt=$[$attempt+1]
+	sleep 1
+    done
+}
+
+# Garbage collect old rkt containers on exit
+trap gc EXIT
+
+if [[ ! $(${RKT_BINARY} image list | grep ${MOUNTER_IMAGE}) ]]; then
+        ${RKT_BINARY} fetch --insecure-options=image file://${MOUNTER_ACI}
+fi
+
+echo "Running mount using a rkt fly container"
+
+${RKT_BINARY} run --stage1-path=${STAGE1_ACI} \
+	--insecure-options=image \
+	--volume=kubelet,kind=host,source=/var/lib/kubelet,readOnly=false,recursive=true \
+	--mount volume=kubelet,target=/var/lib/kubelet \
+	${MOUNTER_IMAGE} --user=${MOUNTER_USER} --exec /bin/mount -- "$@"
+
+echo "Successfully ran mount using a rkt fly container"
diff --git a/cluster/gce/gci/mounter/stage-upload.sh b/cluster/gce/gci/mounter/stage-upload.sh
@@ -0,0 +1,86 @@
+#!/bin/sh
+
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Due to the GCE custom metadata size limit, we split the entire script into two
+# files configure.sh and configure-helper.sh. The functionality of downloading
+# kubernetes configuration, manifests, docker images, and binary files are
+# put in configure.sh, which is uploaded via GCE custom metadata.
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+RKT_VERSION="v1.18.0"
+DOCKER2ACI_VERSION="v0.13.0"
+MOUNTER_VERSION=$1
+DOCKER_IMAGE=docker://$2
+MOUNTER_ACI_IMAGE=gci-mounter-${MOUNTER_VERSION}.aci
+RKT_GCS_DIR=gs://kubernetes-release/rkt/
+MOUNTER_GCS_DIR=gs://kubernetes-release/gci-mounter/
+
+TMPDIR=/tmp
+# Setup a working directory
+DOWNLOAD_DIR=$(mktemp --tmpdir=${TMPDIR} -d gci-mounter-build.XXXXXXXXXX)
+
+# Setup a staging directory
+STAGING_DIR=$(mktemp --tmpdir=${TMPDIR} -d gci-mounter-staging.XXXXXXXXXX)
+RKT_DIR=${STAGING_DIR}/${RKT_VERSION}
+ACI_DIR=${STAGING_DIR}/gci-mounter
+CWD=${PWD}
+
+# Cleanup the temporary directories
+function cleanup {
+    rm -rf ${DOWNLOAD_DIR}
+    rm -rf ${STAGING_DIR}
+    cd ${CWD}
+}
+
+# Delete temporary directories on exit
+trap cleanup EXIT
+
+mkdir ${RKT_DIR}
+mkdir ${ACI_DIR}
+
+# Download rkt
+cd ${DOWNLOAD_DIR}
+echo "Downloading rkt ${RKT_VERSION}"
+wget "https://github.com/coreos/rkt/releases/download/${RKT_VERSION}/rkt-${RKT_VERSION}.tar.gz" &> /dev/null
+echo "Extracting rkt ${RKT_VERSION}"
+tar xzf rkt-${RKT_VERSION}.tar.gz
+
+# Stage rkt into working directory
+cp rkt-${RKT_VERSION}/rkt ${RKT_DIR}/rkt
+cp rkt-${RKT_VERSION}/stage1-fly.aci ${RKT_DIR}/
+
+# Convert docker image to aci and stage it
+echo "Downloading docker2aci ${DOCKER2ACI_VERSION}"
+wget "https://github.com/appc/docker2aci/releases/download/${DOCKER2ACI_VERSION}/docker2aci-${DOCKER2ACI_VERSION}.tar.gz" &> /dev/null
+echo "Extracting docker2aci ${DOCKER2ACI_VERSION}"
+tar xzf docker2aci-${DOCKER2ACI_VERSION}.tar.gz
+ACI_IMAGE=$(${DOWNLOAD_DIR}/docker2aci-${DOCKER2ACI_VERSION}/docker2aci ${DOCKER_IMAGE} 2>/dev/null | tail -n 1)
+cp ${ACI_IMAGE} ${ACI_DIR}/${MOUNTER_ACI_IMAGE}
+
+# Upload the contents to gcs
+echo "Uploading rkt artifacts in ${RKT_DIR} to ${RKT_GCS_DIR}"
+gsutil cp -R ${RKT_DIR} ${RKT_GCS_DIR}
+echo "Uploading gci mounter ACI in ${ACI_DIR} to ${MOUNTER_GCS_DIR}"
+gsutil cp ${ACI_DIR}/${MOUNTER_ACI_IMAGE} ${MOUNTER_GCS_DIR}
+
+echo "Upload completed"
+echo "Update rkt, stag1-fly.aci & gci-mounter ACI versions and SHA1 in cluster/gce/gci/configure.sh"
+echo "${RKT_VERSION}/rkt sha1: $(sha1sum ${RKT_DIR}/rkt)"
+echo "${RKT_VERSION}/stage1-fly.aci sha1: $(sha1sum ${RKT_DIR}/stage1-fly.aci)"
+echo "${MOUNTER_ACI_IMAGE} hash: $(sha1sum ${ACI_DIR}/${MOUNTER_ACI_IMAGE})"
diff --git a/cluster/gce/gci/node.yaml b/cluster/gce/gci/node.yaml
@@ -34,6 +34,7 @@ write_files:
       Type=oneshot
       RemainAfterExit=yes
       ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/configure-helper.sh
+      ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/mounter
       ExecStart=/home/kubernetes/bin/configure-helper.sh
 
       [Install]

diff --git a/cmd/kube-proxy/app/conntrack.go b/cmd/kube-proxy/app/conntrack.go
@@ -89,7 +89,7 @@ func (realConntracker) setIntSysCtl(name string, value int) error {
 func isSysFSWritable() (bool, error) {
 	const permWritable = "rw"
 	const sysfsDevice = "sysfs"
-	m := mount.New()
+	m := mount.New("" /* default mount path */)
 	mountPoints, err := m.List()
 	if err != nil {
 		glog.Errorf("failed to list mount points: %v", err)

diff --git a/cmd/kubelet/app/options/options.go b/cmd/kubelet/app/options/options.go
@@ -171,6 +171,7 @@ func (s *KubeletServer) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.LockFilePath, "lock-file", s.LockFilePath, "<Warning: Alpha feature> The path to file for kubelet to use as a lock file.")
 	fs.BoolVar(&s.ExitOnLockContention, "exit-on-lock-contention", s.ExitOnLockContention, "Whether kubelet should exit upon lock-file contention.")
 	fs.StringVar(&s.RktPath, "rkt-path", s.RktPath, "Path of rkt binary. Leave empty to use the first rkt in $PATH.  Only used if --container-runtime='rkt'.")
+	fs.StringVar(&s.ExperimentalMounterPath, "experimental-mounter-path", s.ExperimentalMounterPath, "[Experimental] Path of mounter binary. Leave empty to use the default mount.")
 	fs.StringVar(&s.RktAPIEndpoint, "rkt-api-endpoint", s.RktAPIEndpoint, "The endpoint of the rkt API service to communicate with. Only used if --container-runtime='rkt'.")
 	fs.StringVar(&s.RktStage1Image, "rkt-stage1-image", s.RktStage1Image, "image to use as stage1. Local paths and http/https URLs are supported. If empty, the 'stage1.aci' in the same directory as '--rkt-path' will be used.")
 	fs.MarkDeprecated("rkt-stage1-image", "Will be removed in a future version. The default stage1 image will be specified by the rkt configurations, see https://github.com/coreos/rkt/blob/master/Documentation/configuration.md for more details.")

diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go
@@ -116,7 +116,7 @@ func UnsecuredKubeletDeps(s *options.KubeletServer) (*kubelet.KubeletDeps, error
 		return nil, err
 	}
 
-	mounter := mount.New()
+	mounter := mount.New(s.ExperimentalMounterPath)
 	var writer kubeio.Writer = &kubeio.StdWriter{}
 	if s.Containerized {
 		glog.V(2).Info("Running kubelet in containerized mode (experimental)")

diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt
@@ -177,6 +177,7 @@ experimental-allowed-unsafe-sysctls
 experimental-bootstrap-kubeconfig
 experimental-flannel-overlay
 experimental-keystone-url
+experimental-mounter-path
 experimental-nvidia-gpus
 experimental-prefix
 external-hostname

diff --git a/pkg/apis/componentconfig/types.go b/pkg/apis/componentconfig/types.go
@@ -305,6 +305,8 @@ type KubeletConfiguration struct {
 	// rktPath is the path of rkt binary. Leave empty to use the first rkt in
 	// $PATH.
 	RktPath string `json:"rktPath,omitempty"`
+	// experimentalMounterPath is the path of mounter binary. Leave empty to use the default mount path
+	ExperimentalMounterPath string `json:"experimentalMounterPath,omitempty"`
 	// rktApiEndpoint is the endpoint of the rkt API service to communicate with.
 	RktAPIEndpoint string `json:"rktAPIEndpoint,omitempty"`
 	// rktStage1Image is the image to use as stage1. Local paths and

diff --git a/pkg/apis/componentconfig/v1alpha1/types.go b/pkg/apis/componentconfig/v1alpha1/types.go
@@ -360,6 +360,9 @@ type KubeletConfiguration struct {
 	// rktPath is the  path of rkt binary. Leave empty to use the first rkt in
 	// $PATH.
 	RktPath string `json:"rktPath"`
+	// experimentalMounterPath is the path to mounter binary. If not set, kubelet will attempt to use mounter
+	// binary that is available via $PATH,
+	ExperimentalMounterPath string `json:"experimentalMounterPath,omitempty"`
 	// rktApiEndpoint is the endpoint of the rkt API service to communicate with.
 	RktAPIEndpoint string `json:"rktAPIEndpoint"`
 	// rktStage1Image is the image to use as stage1. Local paths and

diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.conversion.go
@@ -255,6 +255,7 @@ func autoConvert_v1alpha1_KubeletConfiguration_To_componentconfig_KubeletConfigu
 	out.RemoteImageEndpoint = in.RemoteImageEndpoint
 	out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
 	out.RktPath = in.RktPath
+	out.ExperimentalMounterPath = in.ExperimentalMounterPath
 	out.RktAPIEndpoint = in.RktAPIEndpoint
 	out.RktStage1Image = in.RktStage1Image
 	if err := api.Convert_Pointer_string_To_string(&in.LockFilePath, &out.LockFilePath, s); err != nil {
@@ -435,6 +436,7 @@ func autoConvert_componentconfig_KubeletConfiguration_To_v1alpha1_KubeletConfigu
 	out.RemoteImageEndpoint = in.RemoteImageEndpoint
 	out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
 	out.RktPath = in.RktPath
+	out.ExperimentalMounterPath = in.ExperimentalMounterPath
 	out.RktAPIEndpoint = in.RktAPIEndpoint
 	out.RktStage1Image = in.RktStage1Image
 	if err := api.Convert_string_To_Pointer_string(&in.LockFilePath, &out.LockFilePath, s); err != nil {

diff --git a/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/componentconfig/v1alpha1/zz_generated.deepcopy.go
@@ -263,6 +263,7 @@ func DeepCopy_v1alpha1_KubeletConfiguration(in interface{}, out interface{}, c *
 		out.RemoteImageEndpoint = in.RemoteImageEndpoint
 		out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
 		out.RktPath = in.RktPath
+		out.ExperimentalMounterPath = in.ExperimentalMounterPath
 		out.RktAPIEndpoint = in.RktAPIEndpoint
 		out.RktStage1Image = in.RktStage1Image
 		if in.LockFilePath != nil {

diff --git a/pkg/apis/componentconfig/zz_generated.deepcopy.go b/pkg/apis/componentconfig/zz_generated.deepcopy.go
@@ -276,6 +276,7 @@ func DeepCopy_componentconfig_KubeletConfiguration(in interface{}, out interface
 		out.RemoteImageEndpoint = in.RemoteImageEndpoint
 		out.RuntimeRequestTimeout = in.RuntimeRequestTimeout
 		out.RktPath = in.RktPath
+		out.ExperimentalMounterPath = in.ExperimentalMounterPath
 		out.RktAPIEndpoint = in.RktAPIEndpoint
 		out.RktStage1Image = in.RktStage1Image
 		out.LockFilePath = in.LockFilePath

diff --git a/pkg/kubemark/hollow_kubelet.go b/pkg/kubemark/hollow_kubelet.go
@@ -72,7 +72,7 @@ func NewHollowKubelet(
 		TLSOptions:        nil,
 		OOMAdjuster:       oom.NewFakeOOMAdjuster(),
 		Writer:            &kubeio.StdWriter{},
-		Mounter:           mount.New(),
+		Mounter:           mount.New("" /* default mount path */),
 	}
 
 	return &HollowKubelet{