UPSTREAM: <carry>: Enable build test and verify

.gitignore

@@ -127,3 +127,6 @@ kubernetes.tar.gz
 
 # generated by verify-vendor.sh
 vendordiff.patch
+
+# Ignore openshift source archives produced as part of rpm build
+openshift*.tar.gz

build/run.sh

@@ -25,6 +25,12 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "$KUBE_ROOT/build/common.sh"
 
+# Allow running without docker (e.g. in openshift ci)
+if [[ "${OS_RUN_WITHOUT_DOCKER:-}" ]]; then
+  "${@}"
+  exit 0
+fi
+
 KUBE_RUN_COPY_OUTPUT="${KUBE_RUN_COPY_OUTPUT:-y}"
 
 kube::build::verify_prereqs

hack/make-rules/verify.sh

@@ -46,6 +46,18 @@ EXCLUDED_PATTERNS+=(
   "verify-publishing-bot.py"         # Verifies the upstream rules, which are not maintained in o/k
   )
 
+# Excluded checks for openshift/kubernetes fork
+EXCLUDED_PATTERNS+=(
+  "verify-boilerplate.sh"            # Carries do not require boilerplate
+  "verify-bazel.sh"                  # Bazel is not used downstream
+  "verify-golint.sh"                 # TODO(marun) Cleanup carried code
+  "verify-generated-files-remake.sh" # TODO(marun) Should this be passing?
+  "verify-no-vendor-cycles.sh"       # Incompatible with the way many carries are specified
+  "verify-vendor.sh"                 # Wants to update pins to newer versions than used upstream
+  "verify-publishing-bot.py"         # Verifies the upstream rules, which are not maintained in o/k
+  )
+
+
 # Exclude typecheck in certain cases, if they're running in a separate job.
 if [[ ${EXCLUDE_TYPECHECK:-} =~ ^[yY]$ ]]; then
   EXCLUDED_PATTERNS+=(

hack/update-vendor.sh

@@ -99,10 +99,10 @@ function group_replace_directives() {
      /^replace [(]/      { inreplace=1; next                   }
      inreplace && /^[)]/ { inreplace=0; next                   }
      inreplace           { print > \"${go_mod_replace}\"; next }
-     
+
      # print ungrouped replace directives with the replace directive trimmed
      /^replace [^(]/ { sub(/^replace /,\"\"); print > \"${go_mod_replace}\"; next }
-     
+
      # otherwise print to the noreplace file
      { print > \"${go_mod_noreplace}\" }
   " < go.mod
@@ -136,7 +136,7 @@ function add_generated_comments() {
     echo ""
     cat "${go_mod_nocomments}"
    } > go.mod
-  
+
   # Format
   go mod edit -fmt
 }
@@ -232,7 +232,7 @@ while IFS= read -r repo; do
       go list -tags=tools all
     } >> "${LOG_FILE}" 2>&1
 
-    # capture module dependencies 
+    # capture module dependencies
     go list -m -f '{{if not .Main}}{{.Path}}{{end}}' all > "${tmp_go_deps}"
 
     # restore the original go.mod file
@@ -255,13 +255,13 @@ for repo in $(tsort "${TMP_DIR}/tidy_deps.txt"); do
 
     # prune replace directives that pin to the naturally selected version.
     # do this before tidying, since tidy removes unused modules that
-    # don't provide any relevant packages, which forgets which version of the 
+    # don't provide any relevant packages, which forgets which version of the
     # unused transitive dependency we had a require directive for,
     # and prevents pruning the matching replace directive after tidying.
     go list -m -json all |
-      jq -r 'select(.Replace != null) | 
-             select(.Path == .Replace.Path) | 
-             select(.Version == .Replace.Version) | 
+      jq -r 'select(.Replace != null) |
+             select(.Path == .Replace.Path) |
+             select(.Version == .Replace.Version) |
              "-dropreplace \(.Replace.Path)"' |
     xargs -L 100 go mod edit -fmt
 
@@ -285,9 +285,9 @@ $(go mod why "${loopback_deps[@]}")"
 
     # prune replace directives that pin to the naturally selected version
     go list -m -json all |
-      jq -r 'select(.Replace != null) | 
-             select(.Path == .Replace.Path) | 
-             select(.Version == .Replace.Version) | 
+      jq -r 'select(.Replace != null) |
+             select(.Path == .Replace.Path) |
+             select(.Version == .Replace.Version) |
              "-dropreplace \(.Replace.Path)"' |
     xargs -L 100 go mod edit -fmt
 
@@ -296,6 +296,15 @@ done
 echo "=== tidying root" >> "${LOG_FILE}"
 go mod tidy >>"${LOG_FILE}" 2>&1
 
+# disallow transitive dependencies on k8s.io/kubernetes
+loopback_deps=()
+kube::util::read-array loopback_deps < <(go mod graph | grep ' k8s.io/kubernetes' || true)
+# Allow apiserver-library-go to vendor k8s.io/kubernetes
+if [[ -n ${loopback_deps[*]:+"${loopback_deps[*]}"} && ! "${loopback_deps[*]}" =~ github.com/openshift/apiserver-library-go ]]; then
+  kube::log::error "Disallowed transitive k8s.io/kubernetes dependencies exist via the following imports:"
+  kube::log::error "${loopback_deps[@]}"
+  exit 1
+fi
 
 # Phase 6: add generated comments to go.mod files
 kube::log::status "go.mod: adding generated comments"

openshift-hack/build-go.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+STARTTIME=$(date +%s)
+
+# shellcheck source=openshift-hack/lib/init.sh
+source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh"
+
+pushd "${OS_ROOT}" > /dev/null || exit 1
+  make all WHAT='cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-scheduler cmd/kubelet'
+popd > /dev/null || exit 1
+
+os::build::version::git_vars
+
+if [[ "${OS_GIT_TREE_STATE:-dirty}" == "clean"  ]]; then
+  # only when we are building from a clean state can we claim to
+  # have created a valid set of binaries that can resemble a release
+  mkdir -p "${OS_OUTPUT_RELEASEPATH}"
+  echo "${OS_GIT_COMMIT}" > "${OS_OUTPUT_RELEASEPATH}/.commit"
+fi
+
+ret=$?; ENDTIME=$(date +%s); echo "$0 took $((ENDTIME - STARTTIME)) seconds"; exit "$ret"

openshift-hack/build-rpms.sh

@@ -1,10 +1,13 @@
 #!/usr/bin/env bash
 
 # This script generates release zips and RPMs into _output/releases.
-# All build dependencies are required on the host.
-# We will be running `hack/build-cross.sh` under the covers,
-# so we transitively consume all of the relevant envars.
-source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
+# All build dependencies are required on the host.  The build will be
+# performed by the upstream makefile called from the spec file.
+# shellcheck source=openshift-hack/lib/init.sh
+source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh"
+
+# Only build linux by default. Clearing this value will build all platforms
+OS_ONLY_BUILD_PLATFORMS="${OS_ONLY_BUILD_PLATFORMS:-linux/amd64}"
 
 function cleanup() {
 	return_code=$?
@@ -18,25 +21,16 @@ dirty="$( if [[ -d "${OS_OUTPUT}" ]]; then echo '1'; fi )"
 
 os::util::ensure::system_binary_exists rpmbuild
 os::util::ensure::system_binary_exists createrepo
-os::build::setup_env
 
-if [[ "${OS_ONLY_BUILD_PLATFORMS:-}" == 'linux/amd64' ]]; then
-	# when the user is asking for only Linux binaries, we will
-	# furthermore not build cross-platform clients
-	make_redistributable=0
-else
-	make_redistributable=1
-fi
 if [[ -n "${OS_BUILD_SRPM-}" ]]; then
 	srpm="a"
 else
 	srpm="b"
 fi
 
-
 os::build::rpm::get_nvra_vars
 
-OS_RPM_SPECFILE="$( find "${OS_ROOT}" -name *.spec )"
+OS_RPM_SPECFILE="$( find "${OS_ROOT}" -name '*.spec' )"
 OS_RPM_NAME="$( rpmspec -q --qf '%{name}\n' "${OS_RPM_SPECFILE}" | head -1 )"
 
 os::log::info "Building release RPMs for ${OS_RPM_SPECFILE} ..."
@@ -50,16 +44,15 @@ if [[ -n "${dirty}" && "${OS_GIT_TREE_STATE}" == "dirty" ]]; then
 	os::log::warning "Repository is not clean, performing fast build and reusing _output"
 
 	# build and output from source to destination
+	rm -rf "${rpm_tmp_dir}"
 	mkdir -p "${rpm_tmp_dir}"
-	ln -fs "${OS_ROOT}" "${rpm_tmp_dir}/SOURCES"
-	ln -fs "${OS_ROOT}" "${rpm_tmp_dir}/BUILD"
-
+	ln -fns "${OS_ROOT}" "${rpm_tmp_dir}/SOURCES"
+	ln -fns "${OS_ROOT}" "${rpm_tmp_dir}/BUILD"
 	rpmbuild -bb "${OS_RPM_SPECFILE}" \
 		--define "_sourcedir ${rpm_tmp_dir}/SOURCES" \
 		--define "_builddir ${rpm_tmp_dir}/BUILD" \
 		--define "skip_prep 1" \
 		--define "skip_dist ${SKIP_DIST:-1}" \
-		--define "make_redistributable ${make_redistributable}" \
 		--define "version ${OS_RPM_VERSION}" \
 		--define "release ${OS_RPM_RELEASE}" \
 		--define "commit ${OS_GIT_COMMIT}" \
@@ -70,6 +63,7 @@ if [[ -n "${dirty}" && "${OS_GIT_TREE_STATE}" == "dirty" ]]; then
 	mv -f "${rpm_tmp_dir}"/RPMS/*/*.rpm "${OS_OUTPUT_RPMPATH}"
 
 else
+	rm -rf "${rpm_tmp_dir}/SOURCES"
 	mkdir -p "${rpm_tmp_dir}/SOURCES"
 	tar czf "${rpm_tmp_dir}/SOURCES/${OS_RPM_NAME}-${OS_RPM_VERSION}.tar.gz" \
 		--owner=0 --group=0 \
@@ -79,7 +73,6 @@ else
 
 	rpmbuild -b${srpm} "${OS_RPM_SPECFILE}" \
 		--define "skip_dist ${SKIP_DIST:-1}" \
-		--define "make_redistributable ${make_redistributable}" \
 		--define "version ${OS_RPM_VERSION}" \
 		--define "release ${OS_RPM_RELEASE}" \
 		--define "commit ${OS_GIT_COMMIT}" \
@@ -106,7 +99,7 @@ else
 		mv "${output_directory}"/* "${OS_OUTPUT}"
 	else
 		cp -R "${output_directory}"/* "${OS_OUTPUT}"
-		rm -rf "${output_directory}"/*
+		rm -rf "${output_directory:?}"/*
 	fi
 
 	mkdir -p "${OS_OUTPUT_RPMPATH}"

openshift-hack/conformance-k8s.sh

@@ -6,7 +6,7 @@
 #
 # * all nodes that users can run workloads under marked as schedulable
 #
-source "$(dirname "${BASH_SOURCE}")/../../hack/lib/init.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh"
 
 # Check inputs
 if [[ -z "${KUBECONFIG-}" ]]; then
@@ -16,7 +16,7 @@ test_report_dir="${ARTIFACT_DIR}"
 mkdir -p "${test_report_dir}"
 
 cat <<END > "${test_report_dir}/README.md"
-This conformance report is generated by the OpenShift CI infrastructure. The canonical source location for this test script is located at https://github.com/openshift/origin/blob/master/test/extended/conformance-k8s.sh
+This conformance report is generated by the OpenShift CI infrastructure. The canonical source location for this test script is located at https://github.com/openshift/kubernetes/blob/master/openshift-hack/conformance-k8s.sh
 
 This file was generated by:
 
@@ -32,33 +32,18 @@ To recreate these results
 
 3. Clone the OpenShift source repository and change to that directory:
 
-    git clone https://github.com/openshift/origin.git
-    cd origin
+    git clone https://github.com/openshift/kubernetes.git
+    cd kubernetes
 
 4. Place the \`oc\` binary for that cluster in your PATH
 5. Run the conformance test:
 
-    test/extended/conformance-k8s.sh
+    openshift-hack/conformance-k8s.sh
 
 Nightly conformance tests are run against release branches and reported https://openshift-gce-devel.appspot.com/builds/origin-ci-test/logs/periodic-ci-origin-conformance-k8s/
 END
 
-version="${KUBERNETES_VERSION:-release-1.18}"
-kubernetes="${KUBERNETES_ROOT:-${OS_ROOT}/../../../k8s.io/kubernetes}"
-if [[ -d "${kubernetes}" ]]; then
-  git fetch origin --tags
-else
-  if [[ -n "${KUBERNETES_ROOT-}" ]]; then
-    os::log::fatal "Cannot find Kubernetes source directory, set KUBERNETES_ROOT"
-  fi
-  kubernetes="${OS_ROOT}/_output/components/kubernetes"
-  if [[ ! -d "${kubernetes}" ]]; then
-    mkdir -p "$( dirname "${kubernetes}" )"
-    os::log::info "Cloning Kubernetes source"
-    git clone "https://github.com/kubernetes/kubernetes.git" -b "${version}" "${kubernetes}" # --depth=1 unfortunately we need history info as well
-  fi
-fi
-
+version="release-1.18"
 os::log::info "Running Kubernetes conformance suite for ${version}"
 
 # Execute OpenShift prerequisites
@@ -71,12 +56,10 @@ unschedulable="$( ( oc get nodes -o name -l 'node-role.kubernetes.io/master'; )
 # TODO: undo these operations
 
 # Execute Kubernetes prerequisites
-pushd "${kubernetes}" > /dev/null
-git checkout "${version}"
 make WHAT=cmd/kubectl
 make WHAT=test/e2e/e2e.test
-make WHAT=vendor/github.com/onsi/ginkgo/ginkgo
-export PATH="${kubernetes}/_output/local/bin/$( os::build::host_platform ):${PATH}"
+PATH="${OS_ROOT}/_output/local/bin/$( os::build::host_platform ):${PATH}"
+export PATH
 
 kubectl version  > "${test_report_dir}/version.txt"
 echo "-----"    >> "${test_report_dir}/version.txt"
@@ -86,21 +69,25 @@ oc version      >> "${test_report_dir}/version.txt"
 
 rc=0
 
-ginkgo \
-  -nodes 1 -noColor '-focus=(\[Conformance\].*\[Serial\]|\[Serial\].*\[Conformance\])' $( which e2e.test ) -- \
+e2e_test="$( which e2e.test )"
+
+# shellcheck disable=SC2086
+${e2e_test} \
+  -num-nodes 1 -ginkgo.noColor '-ginkgo.focus=(\[Conformance\].*\[Serial\]|\[Serial\].*\[Conformance\])' \
   -report-dir "${test_report_dir}" \
   -allowed-not-ready-nodes ${unschedulable} \
   2>&1 | tee -a "${test_report_dir}/e2e.log" || rc=1
 
-rename -v junit_ junit_serial_ ${test_report_dir}/junit*.xml
+rename -v junit_ junit_serial_ "${test_report_dir}"/junit*.xml
 
-ginkgo \
-  -nodes 4 -noColor '-skip=\[Serial\]' '-focus=\[Conformance\]' $( which e2e.test ) -- \
+# shellcheck disable=SC2086
+${e2e_test} \
+  -num-nodes 4 -ginkgo.noColor '-ginkgo.skip=\[Serial\]' '-ginkgo.focus=\[Conformance\]' \
   -report-dir "${test_report_dir}" \
   -allowed-not-ready-nodes ${unschedulable} \
   2>&1 | tee -a "${test_report_dir}/e2e.log" || rc=1
 
 echo
 echo "Run complete, results in ${test_report_dir}"
 
-exit $rc
+exit $rc

openshift-hack/images/hyperkube/Dockerfile.rhel

@@ -1,14 +1,13 @@
 FROM registry.svc.ci.openshift.org/ocp/builder:golang-1.13 AS builder
-WORKDIR /go/src/github.com/openshift/origin
+WORKDIR /go/src/github.com/openshift/kubernetes
 COPY . .
-RUN for p in vendor/k8s.io/kubernetes/cmd/kube-apiserver vendor/k8s.io/kubernetes/cmd/kube-controller-manager \
-    vendor/k8s.io/kubernetes/cmd/kube-scheduler vendor/k8s.io/kubernetes/cmd/kubelet; do make build WHAT=$p; done && \
+RUN make WHAT='cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-scheduler cmd/kubelet' && \
     mkdir -p /tmp/build && \
-    cp images/hyperkube/hyperkube /tmp/build && \
-    cp /go/src/github.com/openshift/origin/_output/local/bin/linux/$(go env GOARCH)/{kube-apiserver,kube-controller-manager,kube-scheduler,kubelet} \
+    cp openshift-hack/images/hyperkube/hyperkube /tmp/build && \
+    cp /go/src/github.com/openshift/kubernetes/_output/local/bin/linux/$(go env GOARCH)/{kube-apiserver,kube-controller-manager,kube-scheduler,kubelet} \
     /tmp/build
 
-FROM registry.svc.ci.openshift.org/ocp/4.2:base
+FROM registry.svc.ci.openshift.org/ocp/4.5:base
 RUN yum install -y --setopt=tsflags=nodocs --setopt=skip_missing_names_on_install=False iproute && yum clean all
 COPY --from=builder /tmp/build/* /usr/bin/
 LABEL io.k8s.display-name="OpenShift Kubernetes Server Commands" \

openshift-hack/images/tests/Dockerfile.rhel

@@ -1,16 +1,15 @@
-FROM registry.svc.ci.openshift.org/ocp/builder:golang-1.12 AS builder
-WORKDIR /go/src/github.com/openshift/origin
+FROM registry.svc.ci.openshift.org/ocp/builder:golang-1.13 AS builder
+WORKDIR /go/src/github.com/openshift/kubernetes
 COPY . .
-RUN make build WHAT=cmd/openshift-tests; \
+RUN make WHAT=test/e2e/e2e.test; \
     mkdir -p /tmp/build; \
-    cp /go/src/github.com/openshift/origin/_output/local/bin/linux/$(go env GOARCH)/openshift-tests /tmp/build/openshift-tests
+    cp /go/src/github.com/openshift/kubernetes/_output/local/bin/linux/$(go env GOARCH)/e2e.test /tmp/build/k8s-e2e-tests
 
 FROM registry.svc.ci.openshift.org/ocp/4.2:cli
-COPY --from=builder /tmp/build/openshift-tests /usr/bin/
+COPY --from=builder /tmp/build/k8s-e2e-tests /usr/bin/
 RUN yum install --setopt=tsflags=nodocs -y git gzip util-linux && yum clean all && rm -rf /var/cache/yum/* && \
     git config --system user.name test && \
     git config --system user.email test@test.com && \
     chmod g+w /etc/passwd
-LABEL io.k8s.display-name="OpenShift End-to-End Tests" \
-      io.k8s.description="OpenShift is a platform for developing, building, and deploying containerized applications." \
-      io.openshift.tags="openshift,tests,e2e"
+LABEL io.k8s.display-name="Kubernetes End-to-End Tests" \
+      io.openshift.tags="k8s,tests,e2e"

openshift-hack/lib/build/rpm.sh

@@ -17,7 +17,7 @@ function os::build::rpm::get_nvra_vars() {
 	# the package name can be overwritten but is normally 'origin'
 	OS_RPM_ARCHITECTURE="$(uname -i)"
 
-	# we can extract the pacakge version from the build version
+	# we can extract the package version from the build version
 	os::build::version::get_vars
 	if [[ "${OS_GIT_VERSION}" =~ ^v([0-9](\.[0-9]+)*)(.*) ]]; then
 		OS_RPM_VERSION="${BASH_REMATCH[1]}"

openshift-hack/lib/build/version.sh

@@ -67,8 +67,6 @@ function os::build::version::git_vars() {
 		fi
 	fi
 
-	os::build::version::etcd_vars
-	os::build::version::kubernetes_vars
 }
 readonly -f os::build::version::git_vars