Skip to content

Commit

Permalink
Merge pull request kubernetes#16394 from hakman/fix_shellcheck
Browse files Browse the repository at this point in the history 
Fix shellcheck warnings for bootstrap script
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot committed Mar 6, 2024
2 parents 8f29969 + 6e959ba commit 22ce123
Show file tree
Hide file tree
Showing 185 changed files with 3,186 additions and 2,819 deletions.
9 changes: 6 additions & 3 deletions hack/verify-shellcheck.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@ set -o pipefail

# required version for this script, if not installed on the host we will
# use the official docker image instead. keep this in sync with SHELLCHECK_IMAGE
SHELLCHECK_VERSION="0.7.1"
SHELLCHECK_VERSION="0.9.0"
# upstream shellcheck latest stable image as of September 1st, 2020
SHELLCHECK_IMAGE="koalaman/shellcheck-alpine:v0.7.1@sha256:6093ec552a8c918483dca0544d44ad56a43c4dbf1d85447f2316f8e3b15e3cb6"
SHELLCHECK_IMAGE="koalaman/shellcheck-alpine:v0.9.0@sha256:e19ed93c22423970d56568e171b4512c9244fc75dd9114045016b4a0073ac4b7"

# fixed name for the shellcheck docker container so we can reliably clean it up
SHELLCHECK_CONTAINER="k8s-shellcheck"
Expand All @@ -36,6 +36,8 @@ disabled=(
1090
# this lint prefers command -v to which, they are not the same
2230
# this lint tries to open ./common.sh and fails
1091
)
# comma separate for passing to shellcheck
join_by() {
Expand Down Expand Up @@ -71,7 +73,8 @@ cd "${KOPS_ROOT}"
# - ./vendor* - Vendored code should be fixed upstream instead.
# - ./third_party/*, but re-include ./third_party/forked/* - only code we
# forked should be linted and fixed.
all_shell_scripts=()
# include also output from bootstrap script tests
mapfile -t all_shell_scripts < <(ls -1 pkg/model/tests/data/bootstrapscript_*.txt)
while IFS=$'\n' read -r script;
do git check-ignore -q "$script" || all_shell_scripts+=("$script");
done < <(find . -type f -name "*.sh" \
Expand Down
26 changes: 13 additions & 13 deletions pkg/model/bootstrapscript.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,28 +398,28 @@ func (b *BootstrapScript) createProxyEnv(ps *kops.EgressProxySpec) (string, erro
}

// Set env variables for base environment
buffer.WriteString(`echo "http_proxy=` + httpProxyURL + `" >> /etc/environment` + "\n")
buffer.WriteString(`echo "https_proxy=` + httpProxyURL + `" >> /etc/environment` + "\n")
buffer.WriteString(`echo "no_proxy=` + ps.ProxyExcludes + `" >> /etc/environment` + "\n")
buffer.WriteString(`echo "NO_PROXY=` + ps.ProxyExcludes + `" >> /etc/environment` + "\n")
buffer.WriteString(`{` + "\n")
buffer.WriteString(` echo "http_proxy=` + httpProxyURL + `"` + "\n")
buffer.WriteString(` echo "https_proxy=` + httpProxyURL + `"` + "\n")
buffer.WriteString(` echo "no_proxy=` + ps.ProxyExcludes + `"` + "\n")
buffer.WriteString(` echo "NO_PROXY=` + ps.ProxyExcludes + `"` + "\n")
buffer.WriteString(`} >> /etc/environment` + "\n")

// Load the proxy environment variables
buffer.WriteString("while read in; do export $in; done < /etc/environment\n")
buffer.WriteString("while read -r in; do export \"${in?}\"; done < /etc/environment\n")

// Set env variables for package manager depending on OS Distribution (N/A for Flatcar)
// Note: Nodeup will source the `/etc/environment` file within docker config in the correct location
buffer.WriteString("case `cat /proc/version` in\n")
buffer.WriteString("*[Dd]ebian*)\n")
buffer.WriteString(` echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy ;;` + "\n")
buffer.WriteString("*[Uu]buntu*)\n")
buffer.WriteString(` echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy ;;` + "\n")
buffer.WriteString("case $(cat /proc/version) in\n")
buffer.WriteString("*[Dd]ebian* | *[Uu]buntu*)\n")
buffer.WriteString(` echo "Acquire::http::Proxy \"` + httpProxyURL + `\";" > /etc/apt/apt.conf.d/30proxy ;;` + "\n")
buffer.WriteString("*[Rr]ed[Hh]at*)\n")
buffer.WriteString(` echo "proxy=${http_proxy}" >> /etc/yum.conf ;;` + "\n")
buffer.WriteString(` echo "proxy=` + httpProxyURL + `" >> /etc/yum.conf ;;` + "\n")
buffer.WriteString("esac\n")

// Set env variables for systemd
buffer.WriteString(`echo "DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\"`)
buffer.WriteString(` \"NO_PROXY=${no_proxy}\" \"no_proxy=${no_proxy}\""`)
buffer.WriteString(`echo "DefaultEnvironment=\"http_proxy=` + httpProxyURL + `\" \"https_proxy=` + httpProxyURL + `\"`)
buffer.WriteString(` \"NO_PROXY=` + ps.ProxyExcludes + `\" \"no_proxy=` + ps.ProxyExcludes + `\""`)
buffer.WriteString(" >> /etc/systemd/system.conf\n")

// Restart stuff
Expand Down
2 changes: 1 addition & 1 deletion pkg/model/bootstrapscript_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func Test_ProxyFunc(t *testing.T) {
t.Fatalf("script cannot be empty")
}

if !strings.HasPrefix(script, "echo \"http_proxy=http://example.com:80\" >> /etc/environment") {
if !strings.Contains(script, "echo \"http_proxy=http://example.com:80\"") {
t.Fatalf("script not setting http_proxy properly")
}

Expand Down
32 changes: 17 additions & 15 deletions pkg/model/resources/nodeup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,11 @@ function ensure-install-dir() {
# Retry a download until we get it. args: name, sha, urls
download-or-bust() {
echo "== Downloading $1 with hash $2 from $3 =="
local -r file="$1"
local -r hash="$2"
local -r urls=( $(split-commas "$3") )
local -a urls
mapfile -t urls < <(split-commas "$3")
if [[ -f "${file}" ]]; then
if ! validate-hash "${file}" "${hash}"; then
Expand All @@ -79,28 +81,28 @@ download-or-bust() {
while true; do
for url in "${urls[@]}"; do
commands=(
"curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
"wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
"curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
"wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
"curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
"wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10"
"curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
"wget -O ${file} --connect-timeout=20 --tries=6 --wait=10"
)
for cmd in "${commands[@]}"; do
echo "Attempting download with: ${cmd} {url}"
echo "== Downloading ${url} using ${cmd} =="
if ! (${cmd} "${url}"); then
echo "== Download failed with ${cmd} =="
echo "== Failed to download ${url} using ${cmd} =="
continue
fi
if ! validate-hash "${file}" "${hash}"; then
echo "== Hash validation of ${url} failed. Retrying. =="
echo "== Failed to validate hash for ${url} =="
rm -f "${file}"
else
echo "== Downloaded ${url} (SHA256 = ${hash}) =="
echo "== Downloaded ${url} with hash ${hash} =="
return 0
fi
done
done
echo "All downloads failed; sleeping before retrying"
echo "== All downloads failed; sleeping before retrying =="
sleep 60
done
}
Expand All @@ -110,15 +112,15 @@ validate-hash() {
local -r expected="$2"
local actual
actual=$(sha256sum ${file} | awk '{ print $1 }') || true
actual=$(sha256sum "${file}" | awk '{ print $1 }') || true
if [[ "${actual}" != "${expected}" ]]; then
echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} =="
return 1
fi
}
function split-commas() {
echo $1 | tr "," "\n"
echo "$1" | tr "," "\n"
}
function download-release() {
Expand All @@ -142,14 +144,14 @@ function download-release() {
chmod +x nodeup
echo "Running nodeup"
echo "== Running nodeup =="
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 )
}
####################################################################################
/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured =="
{{- if eq GetCloudProvider "digitalocean" }}
# DO has machine-id baked into the image and journald should be flushed
Expand Down
56 changes: 29 additions & 27 deletions pkg/model/tests/data/bootstrapscript_0.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,20 @@ NODEUP_HASH_ARM64=e525c28a65ff0ce4f95f9e730195b4e67fdcb15ceb1f36b5ad6921a8a4490c
export AWS_REGION=eu-west-1


echo "http_proxy=http://example.com:80" >> /etc/environment
echo "https_proxy=http://example.com:80" >> /etc/environment
echo "no_proxy=" >> /etc/environment
echo "NO_PROXY=" >> /etc/environment
while read in; do export $in; done < /etc/environment
case `cat /proc/version` in
*[Dd]ebian*)
echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy ;;
*[Uu]buntu*)
echo "Acquire::http::Proxy \"${http_proxy}\";" > /etc/apt/apt.conf.d/30proxy ;;
{
echo "http_proxy=http://example.com:80"
echo "https_proxy=http://example.com:80"
echo "no_proxy="
echo "NO_PROXY="
} >> /etc/environment
while read -r in; do export "${in?}"; done < /etc/environment
case $(cat /proc/version) in
*[Dd]ebian* | *[Uu]buntu*)
echo "Acquire::http::Proxy \"http://example.com:80\";" > /etc/apt/apt.conf.d/30proxy ;;
*[Rr]ed[Hh]at*)
echo "proxy=${http_proxy}" >> /etc/yum.conf ;;
echo "proxy=http://example.com:80" >> /etc/yum.conf ;;
esac
echo "DefaultEnvironment=\"http_proxy=${http_proxy}\" \"https_proxy=${http_proxy}\" \"NO_PROXY=${no_proxy}\" \"no_proxy=${no_proxy}\"" >> /etc/systemd/system.conf
echo "DefaultEnvironment=\"http_proxy=http://example.com:80\" \"https_proxy=http://example.com:80\" \"NO_PROXY=\" \"no_proxy=\"" >> /etc/systemd/system.conf
systemctl daemon-reload
systemctl daemon-reexec

Expand All @@ -48,9 +48,11 @@ function ensure-install-dir() {

# Retry a download until we get it. args: name, sha, urls
download-or-bust() {
echo "== Downloading $1 with hash $2 from $3 =="
local -r file="$1"
local -r hash="$2"
local -r urls=( $(split-commas "$3") )
local -a urls
mapfile -t urls < <(split-commas "$3")

if [[ -f "${file}" ]]; then
if ! validate-hash "${file}" "${hash}"; then
Expand All @@ -63,28 +65,28 @@ download-or-bust() {
while true; do
for url in "${urls[@]}"; do
commands=(
"curl -f --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
"wget --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
"curl -f -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
"wget -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
"curl -f --compressed -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
"wget --compression=auto -O ${file} --connect-timeout=20 --tries=6 --wait=10"
"curl -f -Lo ${file} --connect-timeout 20 --retry 6 --retry-delay 10"
"wget -O ${file} --connect-timeout=20 --tries=6 --wait=10"
)
for cmd in "${commands[@]}"; do
echo "Attempting download with: ${cmd} {url}"
echo "== Downloading ${url} using ${cmd} =="
if ! (${cmd} "${url}"); then
echo "== Download failed with ${cmd} =="
echo "== Failed to download ${url} using ${cmd} =="
continue
fi
if ! validate-hash "${file}" "${hash}"; then
echo "== Hash validation of ${url} failed. Retrying. =="
echo "== Failed to validate hash for ${url} =="
rm -f "${file}"
else
echo "== Downloaded ${url} (SHA256 = ${hash}) =="
echo "== Downloaded ${url} with hash ${hash} =="
return 0
fi
done
done

echo "All downloads failed; sleeping before retrying"
echo "== All downloads failed; sleeping before retrying =="
sleep 60
done
}
Expand All @@ -94,15 +96,15 @@ validate-hash() {
local -r expected="$2"
local actual

actual=$(sha256sum ${file} | awk '{ print $1 }') || true
actual=$(sha256sum "${file}" | awk '{ print $1 }') || true
if [[ "${actual}" != "${expected}" ]]; then
echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
echo "== File ${file} is corrupted; hash ${actual} doesn't match expected ${expected} =="
return 1
fi
}

function split-commas() {
echo $1 | tr "," "\n"
echo "$1" | tr "," "\n"
}

function download-release() {
Expand All @@ -126,14 +128,14 @@ function download-release() {

chmod +x nodeup

echo "Running nodeup"
echo "== Running nodeup =="
# We can't run in the foreground because of https://github.com/docker/docker/issues/23793
( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 )
}

####################################################################################

/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
/bin/systemd-machine-id-setup || echo "== Failed to initialize the machine ID; ensure machine-id configured =="

echo "== nodeup node config starting =="
ensure-install-dir
Expand Down
Loading

0 comments on commit 22ce123

Please sign in to comment.