Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix #76093, restructure LICENSES file generation #76586

Merged
merged 2 commits into from Nov 13, 2019
Merged

Fix #76093, restructure LICENSES file generation #76586

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d39ac98
76093 restructure LICENSES file generation
JishanXing Sep 21, 2019
a193dfc
76586 Add generated license files
JishanXing Nov 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 0 additions & 4 deletions Godeps/OWNERS
View file
Open in desktop

This file was deleted.

2 changes: 0 additions & 2 deletions build/common.sh
View file
Open in desktop
Expand Up @@ -729,8 +729,6 @@ function kube::build::copy_output() {
--prune-empty-dirs \
--filter='- /_temp/' \
--filter='+ /vendor/' \
--filter='+ /Godeps/' \
--filter='+ /staging/***/Godeps/**' \
thockin marked this conversation as resolved.
Show resolved Hide resolved
--filter='+ /_output/dockerized/bin/**' \
--filter='+ zz_generated.*' \
--filter='+ generated.proto' \
Expand Down
9 changes: 6 additions & 3 deletions build/lib/release.sh
View file
Open in desktop
Expand Up @@ -187,7 +187,8 @@ function kube::release::package_node_tarballs() {
cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
"${release_stage}/node/bin/"

cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/"
cp -R "${KUBE_ROOT}/vendor/ALL_LICENSES" "${release_stage}/"
cp "${KUBE_ROOT}/LICENSE" "${release_stage}/LICENSES"

cp "${RELEASE_TARS}/kubernetes-src.tar.gz" "${release_stage}/"

Expand Down Expand Up @@ -252,7 +253,8 @@ function kube::release::package_server_tarballs() {
cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
"${release_stage}/server/bin/"

cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/"
cp -R "${KUBE_ROOT}/vendor/ALL_LICENSES" "${release_stage}/"
cp "${KUBE_ROOT}/LICENSE" "${release_stage}/LICENSES"

cp "${RELEASE_TARS}/kubernetes-src.tar.gz" "${release_stage}/"

Expand Down Expand Up @@ -565,7 +567,8 @@ EOF

cp -R "${KUBE_ROOT}/docs" "${release_stage}/"
cp "${KUBE_ROOT}/README.md" "${release_stage}/"
cp "${KUBE_ROOT}/Godeps/LICENSES" "${release_stage}/"
cp -R "${KUBE_ROOT}/vendor/ALL_LICENSES" "${release_stage}/"
cp "${KUBE_ROOT}/LICENSE" "${release_stage}/LICENSES"

echo "${KUBE_GIT_VERSION}" > "${release_stage}/version"

Expand Down
14 changes: 12 additions & 2 deletions build/release-tars/BUILD
View file
Open in desktop
Expand Up @@ -69,12 +69,21 @@ pkg_tar(
],
)

filegroup(
name = "all-licenses-files",
srcs = glob([
"vendor/ALL_LICENSES/*",
]),
visibility = ["//visibility:private"],
)

# Included in node and server tarballs.
filegroup(
name = "license-targets",
srcs = [
":all-licenses-files",
":kubernetes-src.tar.gz",
"//:Godeps/LICENSES",
"//:LICENSE",
],
visibility = ["//visibility:private"],
)
Expand Down Expand Up @@ -235,7 +244,8 @@ pkg_tar(
pkg_tar(
name = "kubernetes",
srcs = [
"//:Godeps/LICENSES",
":all-licenses-files",
"//:LICENSE",
"//:README.md",
"//:version",
"//cluster:all-srcs",
Expand Down
2 changes: 1 addition & 1 deletion cluster/gce/gci/configure.sh
View file
Open in desktop
Expand Up @@ -421,7 +421,7 @@ function install-kube-binary-config {
mv "${src_dir}/kubelet" "${KUBE_BIN}"
mv "${src_dir}/kubectl" "${KUBE_BIN}"

mv "${KUBE_HOME}/kubernetes/LICENSES" "${KUBE_HOME}"
mv "${KUBE_HOME}/kubernetes/LICENSE" "${KUBE_HOME}"
Copy link
Member

@liggitt liggitt Nov 13, 2019 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this does not exist in the artifacts consumed by some CI jobs, see https://storage.googleapis.com/kubernetes-jenkins/logs/ci-kubernetes-e2e-gci-gce-alpha-features/1194514172650459137/artifacts/bootstrap-e2e-master/kube-master-installation.log

See #85215 #85214 #85213 #85196

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fix in progress in #85210

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reverting just this line fixed some jobs and failed others. see test results in #85210

That means that some of our build paths produce this file, and some do not.

Reverting this PR in #85220 until we can resolve this issue consistently.

mv "${KUBE_HOME}/kubernetes/kubernetes-src.tar.gz" "${KUBE_HOME}"
fi

Expand Down
2 changes: 1 addition & 1 deletion hack/boilerplate/boilerplate.py
View file
Open in desktop
Expand Up @@ -155,7 +155,7 @@ def file_extension(filename):
return os.path.splitext(filename)[1].split(".")[-1].lower()


skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', 'cluster/env.sh',
thockin marked this conversation as resolved.
Show resolved Hide resolved
skipped_dirs = ['third_party', '_gopath', '_output', '.git', 'cluster/env.sh',
"vendor", "test/e2e/generated/bindata.go", "hack/boilerplate/test",
"staging/src/k8s.io/kubectl/pkg/generated/bindata.go"]

Expand Down
58 changes: 39 additions & 19 deletions hack/update-vendor-licenses.sh
View file
Open in desktop
Expand Up @@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Update the Godeps/LICENSES document.
# Update the vendor/ALL_LICENSES document.
# Generates a table of Godep dependencies and their license.
#
# Usage:
Expand All @@ -24,12 +24,15 @@
# additionally created files into the vendor auto-generated tree.
#
# Run every time a license file is added/modified within /vendor to
# update /Godeps/LICENSES
# update /vendor/ALL_LICENSES

set -o errexit
set -o nounset
set -o pipefail

KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"

export LANG=C
export LC_ALL=C

Expand Down Expand Up @@ -130,8 +133,6 @@ process_content () {
#############################################################################
# MAIN
#############################################################################
KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
source "${KUBE_ROOT}/hack/lib/init.sh"

export GO111MODULE=on

Expand All @@ -153,8 +154,19 @@ fi
LICENSE_ROOT="${LICENSE_ROOT:-${KUBE_ROOT}}"
cd "${LICENSE_ROOT}"

VENDOR_LICENSE_FILE="Godeps/LICENSES"
TMP_LICENSE_FILE="/tmp/Godeps.LICENSES.$$"
kube::util::ensure-temp-dir

# Save the genreated LICENSE file for each package temporarily
TMP_LICENSE_FILE="${KUBE_TEMP}/vendor.LICENSES.$$"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you comment these?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed.


# The directory to save all the LICENSE files
VENDOR_LICENSE_DIR="${LICENSE_ROOT}/vendor/ALL_LICENSES"

# The tmp directory to save all the LICENSE files, will move to VENDOR_LICENSE_DIR
TMP_VENDOR_LICENSE_DIR="${KUBE_TEMP}/vendor.LICENSES.DIR.$$"

mkdir -p "${TMP_VENDOR_LICENSE_DIR}"

DEPS_DIR="vendor"
declare -Ag CONTENT

Expand All @@ -167,7 +179,8 @@ cat "${LICENSE_ROOT}/LICENSE"
echo
echo "= LICENSE $(kube::util::md5 "${LICENSE_ROOT}/LICENSE")"
echo "================================================================================"
) > ${TMP_LICENSE_FILE}
) > "${TMP_LICENSE_FILE}"
mv "${TMP_LICENSE_FILE}" "${TMP_VENDOR_LICENSE_DIR}/LICENSE"

# Loop through every vendored package
for PACKAGE in $(go list -m -json all | jq -r .Path | sort -f); do
Expand All @@ -184,11 +197,10 @@ for PACKAGE in $(go list -m -json all | jq -r .Path | sort -f); do
process_content "${PACKAGE}" COPYRIGHT
process_content "${PACKAGE}" COPYING

# display content
echo
echo "================================================================================"
echo "= ${DEPS_DIR}/${PACKAGE} licensed under: ="
echo
# copy content and throw error message
{
echo "= ${DEPS_DIR}/${PACKAGE} licensed under: ="
echo

file=""
if [[ -n "${CONTENT[${PACKAGE}-LICENSE]-}" ]]; then
Expand All @@ -211,12 +223,20 @@ Options:
__EOF__
exit 9
fi
cat "${file}"

cat "${file}"
echo
echo "= ${file} $(kube::util::md5 "${file}")"
} >> "${TMP_LICENSE_FILE}"

echo
echo "= ${file} $(kube::util::md5 "${file}")"
echo "================================================================================"
echo
done >> ${TMP_LICENSE_FILE}

cat ${TMP_LICENSE_FILE} > ${VENDOR_LICENSE_FILE}
dest_dir="${TMP_VENDOR_LICENSE_DIR}/${PACKAGE}"
mkdir -p "${dest_dir}"
mv "${TMP_LICENSE_FILE}" "${dest_dir}/LICENSE"
done

if [[ -d "${VENDOR_LICENSE_DIR}" ]]; then
rm -r "${VENDOR_LICENSE_DIR}"
fi
mv "${TMP_VENDOR_LICENSE_DIR}" "${VENDOR_LICENSE_DIR}"

5 changes: 2 additions & 3 deletions hack/update-vendor.sh
View file
Open in desktop
Expand Up @@ -340,11 +340,10 @@ kube::log::status "vendor: updating LICENSES file"
hack/update-vendor-licenses.sh >>"${LOG_FILE}" 2>&1

kube::log::status "vendor: creating OWNERS file"
rm -f "Godeps/OWNERS" "vendor/OWNERS"
cat <<__EOF__ > "Godeps/OWNERS"
rm -f "vendor/OWNERS"
cat <<__EOF__ > "vendor/OWNERS"
# See the OWNERS docs at https://go.k8s.io/owners

approvers:
- dep-approvers
__EOF__
cp "Godeps/OWNERS" "vendor/OWNERS"
15 changes: 12 additions & 3 deletions hack/verify-vendor-licenses.sh
View file
Open in desktop
Expand Up @@ -33,16 +33,25 @@ function cleanup {
}
kube::util::trap_add cleanup EXIT

cp -r "${KUBE_ROOT}/Godeps" "${_tmpdir}/Godeps"
#ln -s all subfolder except ALL_LICENSES in vendor
mkdir -p "${_tmpdir}/vendor"
for child in "${KUBE_ROOT}/vendor"/*
do
if [[ "${child}" == "${KUBE_ROOT}/vendor/ALL_LICENSES" ]] ; then
continue
else
ln -s "${child}" "${_tmpdir}/vendor"
fi
done

ln -s "${KUBE_ROOT}/LICENSE" "${_tmpdir}"
ln -s "${KUBE_ROOT}/vendor" "${_tmpdir}"
ln -s "${KUBE_ROOT}/staging" "${_tmpdir}"

# Update vendor Licenses
LICENSE_ROOT="${_tmpdir}" "${KUBE_ROOT}/hack/update-vendor-licenses.sh"

# Compare vendor Licenses
if ! _out="$(diff -Naupr "${KUBE_ROOT}/Godeps/LICENSES" "${_tmpdir}/Godeps/LICENSES")"; then
if ! _out="$(diff -Naupr "${KUBE_ROOT}/vendor/ALL_LICENSES" "${_tmpdir}/vendor/ALL_LICENSES")"; then
thockin marked this conversation as resolved.
Show resolved Hide resolved
echo "Your vendor licenses file is out of date. Run hack/update-vendor-licenses.sh and commit the results." >&2
echo "${_out}" >&2
exit 1
Expand Down
2 changes: 1 addition & 1 deletion hack/verify-vendor.sh
View file
Open in desktop
Expand Up @@ -55,7 +55,7 @@ export GO111MODULE=on

pushd "${_kubetmp}" > /dev/null 2>&1
# Destroy deps in the copy of the kube tree
rm -rf ./Godeps/LICENSES ./vendor
rm -rf ./vendor

# Recreate the vendor tree using the nice clean set we just downloaded
hack/update-vendor.sh
Expand Down
4 changes: 2 additions & 2 deletions staging/publishing/rules-godeps.yaml
View file
Open in desktop
Expand Up @@ -121,7 +121,7 @@ rules:
go build .
# re-create vendor/ and try again
godep restore
rm -rf vendor/ Godeps/
rm -rf vendor/
godep save ./...
go build .
- destination: sample-controller
Expand Down Expand Up @@ -150,7 +150,7 @@ rules:

# re-create vendor/ and try again
godep restore
rm -rf vendor/ Godeps/
rm -rf vendor/
godep save ./...
go build .
- destination: apiextensions-apiserver
Expand Down