Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Distroless] Convert the GCE manifests for master containers. #75624

Merged
merged 1 commit into from
Apr 4, 2019
Merged

[Distroless] Convert the GCE manifests for master containers. #75624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 15 additions & 9 deletions cluster/gce/gci/apiserver_manifest_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,11 +93,12 @@ func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv, kubeEnv s

func TestEncryptionProviderFlag(t *testing.T) {
var (
// command": [
// "/bin/sh", - Index 0
// "-c", - Index 1
// "exec /usr/local/bin/kube-apiserver " - Index 2
execArgsIndex = 2
// command": [
// "/usr/local/bin/kube-apiserver " - Index 0,
// "--flag1=val1", - Index 1,
// "--flag2=val2", - Index 2,
// ...
// "--flagN=valN", - Index N,
encryptionConfigFlag = "--encryption-provider-config"
)

Expand Down Expand Up @@ -131,10 +132,15 @@ func TestEncryptionProviderFlag(t *testing.T) {

c.invokeTest(e, deployHelperEnv)

execArgs := c.pod.Spec.Containers[0].Command[execArgsIndex]
flagIsInArg := strings.Contains(execArgs, encryptionConfigFlag)
flag := fmt.Sprintf("%s=%s", encryptionConfigFlag, e.EncryptionProviderConfigPath)

var flagIsInArg bool
var flag, execArgs string
for _, execArgs = range c.pod.Spec.Containers[0].Args[1:] {
if strings.Contains(execArgs, encryptionConfigFlag) {
flagIsInArg = true
flag = fmt.Sprintf("%s=%s", encryptionConfigFlag, e.EncryptionProviderConfigPath)
break
}
}
switch {
case tc.wantFlag && !flagIsInArg:
t.Fatalf("Got %q,\n want flags to contain %q", execArgs, flag)
Expand Down
41 changes: 36 additions & 5 deletions cluster/gce/gci/configure-helper.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,24 @@ set -o errexit
set -o nounset
set -o pipefail

function convert-manifest-params {
# A helper function to convert the manifest args from a string to a list of
# flag arguments.
# Old format:
# command=["/bin/sh", "-c", "exec KUBE_EXEC_BINARY --param1=val1 --param2-val2"].
# New format:
# command=["KUBE_EXEC_BINARY"] # No shell dependencies.
# args=["--param1=val1", "--param2-val2"]
IFS=' ' read -ra FLAGS <<< "$1"
params=""
for flag in "${FLAGS[@]}"; do
params+="\n\"$flag\","
done
if [ ! -z $params ]; then
echo "${params::-1}" # drop trailing comma
fi
}

function setup-os-params {
# Reset core_pattern. On GCI, the default core_pattern pipes the core dumps to
# /sbin/crash_reporter which is more restrictive in saving crash dumps. So for
Expand Down Expand Up @@ -1850,6 +1868,9 @@ function start-kube-apiserver {
# params is passed by reference, so no "$"
setup-etcd-encryption "${src_file}" params

params+=" --log-file=${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}"
params+=" --logtostderr=false"
params="$(convert-manifest-params "${params}")"
# Evaluate variables.
local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
Expand Down Expand Up @@ -2031,7 +2052,8 @@ function apply-encryption-config() {
function start-kube-controller-manager {
echo "Start kubernetes controller-manager"
create-kubecontrollermanager-kubeconfig
prepare-log-file /var/log/kube-controller-manager.log
local LOG_PATH=/var/log/kube-controller-manager.log
prepare-log-file "${LOG_PATH}"
# Calculate variables and assemble the command line.
local params="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=2"} ${CONTROLLER_MANAGER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}"
params+=" --use-service-account-credentials"
Expand Down Expand Up @@ -2059,7 +2081,7 @@ function start-kube-controller-manager {
params+=" --concurrent-service-syncs=${CONCURRENT_SERVICE_SYNCS}"
fi
if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then
params+=" --allocate-node-cidrs=true"
yuwenma marked this conversation as resolved.
Show resolved Hide resolved
params+=" --allocate-node-cidrs"
elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then
params+=" --allocate-node-cidrs=${ALLOCATE_NODE_CIDRS}"
fi
Expand Down Expand Up @@ -2090,9 +2112,13 @@ function start-kube-controller-manager {
params+=" --pv-recycler-pod-template-filepath-hostpath=$PV_RECYCLER_OVERRIDE_TEMPLATE"
fi
if [[ -n "${RUN_CONTROLLERS:-}" ]]; then
params+=" --controllers=${RUN_CONTROLLERS}"
# Trim the `RUN_CONTROLLERS` value. This field is quoted which is
# incompatible with the `convert-manifest-params` format.
params+=" --controllers=${RUN_CONTROLLERS//\'}"
fi

params+=" --log-file=${LOG_PATH}"
params+=" --logtostderr=false"
params="$(convert-manifest-params "${params}")"
local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
local container_env=""
if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then
Expand Down Expand Up @@ -2127,7 +2153,8 @@ function start-kube-controller-manager {
function start-kube-scheduler {
echo "Start kubernetes scheduler"
create-kubescheduler-kubeconfig
prepare-log-file /var/log/kube-scheduler.log
local LOG_PATH=/var/log/kube-scheduler.log
prepare-log-file "${LOG_PATH}"

# Calculate variables and set them in the manifest.
params="${SCHEDULER_TEST_LOG_LEVEL:-"--v=2"} ${SCHEDULER_TEST_ARGS:-}"
Expand All @@ -2143,6 +2170,10 @@ function start-kube-scheduler {
params+=" --use-legacy-policy-config"
params+=" --policy-config-file=/etc/srv/kubernetes/kube-scheduler/policy-config"
fi

params+=" --log-file=${LOG_PATH}"
params+=" --logtostderr=false"
params="$(convert-manifest-params "${params}")"
local -r kube_scheduler_docker_tag=$(cat "${KUBE_HOME}/kube-docker-files/kube-scheduler.docker_tag")

# Remove salt comments and replace variables with values.
Expand Down
10 changes: 6 additions & 4 deletions cluster/gce/manifests/kube-apiserver.manifest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,12 @@
}
},
"command": [
"/bin/sh",
"-c",
"exec /usr/local/bin/kube-apiserver {{params}} --allow-privileged={{pillar['allow_privileged']}} 1>>/var/log/kube-apiserver.log 2>&1"
],
"/usr/local/bin/kube-apiserver"
],
"args": [
"--allow-privileged={{pillar['allow_privileged']}}",
yuwenma marked this conversation as resolved.
Show resolved Hide resolved
{{params}}
],
{{container_env}}
"livenessProbe": {
"httpGet": {
Expand Down
9 changes: 5 additions & 4 deletions cluster/gce/manifests/kube-controller-manager.manifest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@
}
},
"command": [
"/bin/sh",
"-c",
"exec /usr/local/bin/kube-controller-manager {{params}} 1>>/var/log/kube-controller-manager.log 2>&1"
],
"/usr/local/bin/kube-controller-manager"
],
"args": [
{{params}}
yuwenma marked this conversation as resolved.
Show resolved Hide resolved
],
{{container_env}}
"livenessProbe": {
"httpGet": {
Expand Down
9 changes: 5 additions & 4 deletions cluster/gce/manifests/kube-scheduler.manifest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,11 @@
}
},
"command": [
"/bin/sh",
"-c",
"exec /usr/local/bin/kube-scheduler {{params}} 1>>/var/log/kube-scheduler.log 2>&1"
],
"/usr/local/bin/kube-scheduler"
],
"args": [
{{params}}
],
"livenessProbe": {
"httpGet": {
"host": "127.0.0.1",
Expand Down