Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix update-vendor.sh logging #118187

Merged
merged 1 commit into from May 22, 2023
Merged

Fix update-vendor.sh logging #118187

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
67 changes: 34 additions & 33 deletions hack/update-vendor.sh
View file
Open in desktop
Expand Up @@ -44,21 +44,23 @@ TMP_DIR="${TMP_DIR:-$(mktemp -d /tmp/update-vendor.XXXX)}"
LOG_FILE="${LOG_FILE:-${TMP_DIR}/update-vendor.log}"
kube::log::status "logfile at ${LOG_FILE}"

# Set up some FDs for this script to use, while capturing everything else to
# the log. NOTHING ELSE should write to $LOG_FILE directly.
exec 11>&1 # Real stdout, use this explicitly
exec 22>&2 # Real stderr, use this explicitly
exec 1>"${LOG_FILE}" # Automatic stdout
exec 2>&1 # Automatic stderr
set -x # Trace this script to stderr

function finish {
ret=$?
if [[ ${ret} != 0 ]]; then
echo "An error has occurred. Please see more details in ${LOG_FILE}"
echo "An error has occurred. Please see more details in ${LOG_FILE}" >&22
fi
exit ${ret}
}
trap finish EXIT

if [ -z "${BASH_XTRACEFD:-}" ]; then
exec 19> "${LOG_FILE}"
export BASH_XTRACEFD="19"
set -x
fi

# ensure_require_replace_directives_for_all_dependencies:
# - ensures all existing 'require' directives have an associated 'replace' directive pinning a version
# - adds explicit 'require' directives for all transitive dependencies
Expand Down Expand Up @@ -178,7 +180,7 @@ function add_generated_comments() {
for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
if [[ ! -f go.mod ]]; then
kube::log::status "go.mod: initialize ${repo}"
kube::log::status "go.mod: initialize ${repo}" >&11
rm -f Godeps/Godeps.json # remove before initializing, staging Godeps are not authoritative
go mod init "k8s.io/${repo}"
go mod edit -fmt
Expand All @@ -187,15 +189,15 @@ for repo in $(kube::util::list_staging_repos); do
done

if [[ ! -f go.mod ]]; then
kube::log::status "go.mod: initialize k8s.io/kubernetes"
kube::log::status "go.mod: initialize k8s.io/kubernetes" >&11
go mod init "k8s.io/kubernetes"
rm -f Godeps/Godeps.json # remove after initializing
fi


# Phase 2: ensure staging repo require/replace directives

kube::log::status "go.mod: update staging references"
kube::log::status "go.mod: update staging references" >&11
# Prune
go mod edit -json \
| jq -r '.Require[]? | select(.Version == "v0.0.0") | "-droprequire \(.Path)"' \
Expand All @@ -217,18 +219,18 @@ kube::util::list_staging_repos \
# pin referenced versions
ensure_require_replace_directives_for_all_dependencies
# resolves/expands references in the root go.mod (if needed)
go mod tidy >>"${LOG_FILE}" 2>&1
go mod tidy
# pin expanded versions
ensure_require_replace_directives_for_all_dependencies
# group require/replace directives
group_directives

# Phase 4: copy root go.mod to staging dirs and rewrite

kube::log::status "go.mod: propagate to staging modules"
kube::log::status "go.mod: propagate to staging modules" >&11
for repo in $(kube::util::list_staging_repos); do
pushd "staging/src/k8s.io/${repo}" >/dev/null 2>&1
echo "=== propagating to ${repo}" >> "${LOG_FILE}"
echo "=== propagating to ${repo}"
# copy root go.mod, changing module name
sed "s#module k8s.io/kubernetes#module k8s.io/${repo}#" \
< "${KUBE_ROOT}/go.mod" \
Expand All @@ -247,7 +249,7 @@ done

# Phase 5: sort and tidy staging components

kube::log::status "go.mod: sorting staging modules"
kube::log::status "go.mod: sorting staging modules" >&11
# tidy staging repos in reverse dependency order.
# the content of dependencies' go.mod files affects what `go mod tidy` chooses to record in a go.mod file.
tidy_unordered="${TMP_DIR}/tidy_unordered.txt"
Expand Down Expand Up @@ -276,7 +278,7 @@ while IFS= read -r repo; do
go list all
echo "=== computing tools imports for ${repo}"
go list -tags=tools all
} >> "${LOG_FILE}" 2>&1
}

# capture module dependencies
go list -m -f '{{if not .Main}}{{.Path}}{{end}}' all > "${tmp_go_deps}"
Expand All @@ -294,10 +296,10 @@ while IFS= read -r repo; do
popd >/dev/null 2>&1
done < "${tidy_unordered}"

kube::log::status "go.mod: tidying"
kube::log::status "go.mod: tidying" >&11
for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do
pushd "${KUBE_ROOT}/staging/src/${repo}" >/dev/null 2>&1
echo "=== tidying ${repo}" >> "${LOG_FILE}"
echo "=== tidying ${repo}"

# prune replace directives that pin to the naturally selected version.
# do this before tidying, since tidy removes unused modules that
Expand All @@ -311,14 +313,13 @@ for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do
"-dropreplace \(.Replace.Path)"' |
xargs -L 100 go mod edit -fmt

go mod tidy -v >>"${LOG_FILE}" 2>&1
go mod tidy -v

# disallow transitive dependencies on k8s.io/kubernetes
loopback_deps=()
kube::util::read-array loopback_deps < <(go list all 2>/dev/null | grep k8s.io/kubernetes/ || true)
if [[ -n ${loopback_deps[*]:+"${loopback_deps[*]}"} ]]; then
kube::log::error "Disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports:
$(go mod why "${loopback_deps[@]}")"
kube::log::error "Disallowed ${repo} -> k8s.io/kubernetes dependencies exist via the following imports: $(go mod why "${loopback_deps[@]}")" >&22 2>&1
exit 1
fi

Expand All @@ -342,8 +343,8 @@ $(go mod why "${loopback_deps[@]}")"

popd >/dev/null 2>&1
done
echo "=== tidying root" >> "${LOG_FILE}"
go mod tidy >>"${LOG_FILE}" 2>&1
echo "=== tidying root"
go mod tidy

# prune unused pinned non-local replace directives
comm -23 \
Expand All @@ -356,13 +357,13 @@ xargs -L 100 go mod edit -fmt
loopback_deps=()
kube::util::read-array loopback_deps < <(go mod graph | grep ' k8s.io/kubernetes' || true)
if [[ -n ${loopback_deps[*]:+"${loopback_deps[*]}"} ]]; then
kube::log::error "Disallowed transitive k8s.io/kubernetes dependencies exist via the following imports:"
kube::log::error "${loopback_deps[@]}"
kube::log::error "Disallowed transitive k8s.io/kubernetes dependencies exist via the following imports:" >&22 2>&1
kube::log::error "${loopback_deps[@]}" >&22 2>&1
exit 1
fi

# Phase 6: add generated comments to go.mod files
kube::log::status "go.mod: adding generated comments"
kube::log::status "go.mod: adding generated comments" >&11
add_generated_comments "
// This is a generated file. Do not edit directly.
// Ensure you've carefully read
Expand All @@ -378,13 +379,13 @@ done


# Phase 7: update internal modules
kube::log::status "vendor: updating internal modules"
hack/update-internal-modules.sh >>"${LOG_FILE}" 2>&1
kube::log::status "vendor: updating internal modules" >&11
hack/update-internal-modules.sh


# Phase 8: rebuild vendor directory
kube::log::status "vendor: running 'go mod vendor'"
go mod vendor >>"${LOG_FILE}" 2>&1
kube::log::status "vendor: running 'go mod vendor'" >&11
go mod vendor

# create a symlink in vendor directory pointing to the staging components.
# This lets other packages and tools use the local staging components as if they were vendored.
Expand All @@ -393,10 +394,10 @@ for repo in $(kube::util::list_staging_repos); do
ln -s "../../staging/src/k8s.io/${repo}" "${KUBE_ROOT}/vendor/k8s.io/${repo}"
done

kube::log::status "vendor: updating vendor/LICENSES"
hack/update-vendor-licenses.sh >>"${LOG_FILE}" 2>&1
kube::log::status "vendor: updating vendor/LICENSES" >&11
hack/update-vendor-licenses.sh

kube::log::status "vendor: creating OWNERS file"
kube::log::status "vendor: creating OWNERS file" >&11
rm -f "vendor/OWNERS"
cat <<__EOF__ > "vendor/OWNERS"
# See the OWNERS docs at https://go.k8s.io/owners
Expand All @@ -410,4 +411,4 @@ reviewers:
- dep-reviewers
__EOF__

kube::log::status "NOTE: don't forget to handle vendor/* and LICENSE/* files that were added or removed"
kube::log::status "NOTE: don't forget to handle vendor/* and LICENSE/* files that were added or removed" >&11