kubectl: add integration test for result reporting

This currently covers two cases:
- "kubectl list" (the regression from kubernetes#107012)
- "kubectl get pods/no-such-pod" (no particular reason except that the output
should be deterministic)

In contrast to some other tests that check for strings inside the
output (run_deprecated_api_tests) or compare after
sorting (run_kubectl_version_tests), stdout, stderr and the return code must
match exactly.

This ensures that there is no extra, unexpected output and that the right
output stream is used.

hack/lib/test.sh

@@ -451,10 +451,16 @@ kube::test::version::diff_assert() {
         return 1
   fi
 
-  sort "${original}" > "${original}.sorted"
-  sort "${latest}" > "${latest}.sorted"
+  if [ "${comparator}" == "exact" ]; then
+      # Skip sorting of file content for exact comparison.
+      cp "${original}" "${original}.sorted"
+      cp "${latest}" "${latest}.sorted"
+  else
+      sort "${original}" > "${original}.sorted"
+      sort "${latest}" > "${latest}.sorted"
+  fi
 
-  if [ "${comparator}" == "eq" ]; then
+  if [ "${comparator}" == "eq" ] || [ "${comparator}" == "exact" ]; then
     if [ "$(diff -iwB "${original}".sorted "${latest}".sorted)" == "" ] ; then
         echo -n "${green}"
         echo "Successful: ${diff_msg}"
@@ -493,3 +499,47 @@ kube::test::version::diff_assert() {
   fi
 }
 
+# Force exact match of kubectl stdout, stderr, and return code.
+# $1: file with actual stdout
+# $2: file with actual stderr
+# $3: the actual return code
+# $4: file with expected stdout
+# $5: file with expected stderr
+# $6: expected return code
+# $7: additional message describing the invocation
+kube::test::results::diff() {
+  local actualstdout=$1
+  local actualstderr=$2
+  local actualcode=$3
+  local expectedstdout=$4
+  local expectedstderr=$5
+  local expectedcode=$6
+  local message=$7
+  local result=0
+
+  if ! kube::test::version::diff_assert "${expectedstdout}" "exact" "${actualstdout}" "stdout for ${message}"; then
+      result=1
+  fi
+  if ! kube::test::version::diff_assert "${expectedstderr}" "exact" "${actualstderr}" "stderr for ${message}"; then
+      result=1
+  fi
+  if [ "${actualcode}" -ne "${expectedcode}" ]; then
+      echo "${bold}${red}"
+      echo "$(kube::test::get_caller): FAIL!"
+      echo "Return code for ${message}"
+      echo "  Expected: ${expectedcode}"
+      echo "  Got:      ${actualcode}"
+      echo "${reset}${red}"
+      caller
+      echo "${reset}"
+      result=1
+  fi
+
+  if [ "${result}" -eq 0 ]; then
+     echo -n "${green}"
+     echo "$(kube::test::get_caller): Successful: ${message}"
+     echo -n "${reset}"
+  fi
+
+  return "$result"
+}

test/cmd/legacy-script.sh

@@ -50,6 +50,7 @@ source "${KUBE_ROOT}/test/cmd/plugins.sh"
 source "${KUBE_ROOT}/test/cmd/proxy.sh"
 source "${KUBE_ROOT}/test/cmd/rbac.sh"
 source "${KUBE_ROOT}/test/cmd/request-timeout.sh"
+source "${KUBE_ROOT}/test/cmd/results.sh"
 source "${KUBE_ROOT}/test/cmd/run.sh"
 source "${KUBE_ROOT}/test/cmd/save-config.sh"
 source "${KUBE_ROOT}/test/cmd/storage.sh"
@@ -436,6 +437,12 @@ runTests() {
 
   record_command run_kubectl_version_tests
 
+  ############################
+  # Kubectl result reporting #
+  ############################
+
+  record_command run_kubectl_results_tests
+
   #######################
   # kubectl config set #
   #######################

test/cmd/results.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+
+############################################################
+# Kubectl result reporting for different failure scenarios #
+############################################################
+run_kubectl_results_tests() {
+  set -o nounset
+  set -o errexit
+
+  kube::log::status "Testing kubectl result output"
+  TEMP="${KUBE_TEMP}"
+  rm -f "${TEMP}/empty"
+  touch "${TEMP}/empty"
+
+  set +o errexit
+  kubectl list >"${TEMP}/actual_stdout" 2>"${TEMP}/actual_stderr"
+  res=$?
+  set -o errexit
+  cat >"${TEMP}/expected_stderr" <<EOF
+error: unknown command "list" for "kubectl"
+
+Did you mean this?
+	get
+	wait
+EOF
+  kube::test::results::diff "${TEMP}/actual_stdout" "${TEMP}/actual_stderr" "$res" "${TEMP}/empty" "${TEMP}/expected_stderr" 1 "kubectl list"
+
+  set +o errexit
+  kubectl get pod/no-such-pod >"${TEMP}/actual_stdout" 2>"${TEMP}/actual_stderr"
+  res=$?
+  set -o errexit
+  cat >"${TEMP}/expected_stderr" <<EOF
+Error from server (NotFound): pods "no-such-pod" not found
+EOF
+  kube::test::results::diff "${TEMP}/actual_stdout" "${TEMP}/actual_stderr" "$res" "${TEMP}/empty" "${TEMP}/expected_stderr" 1 "kubectl get pod/no-such-pod"
+
+  set +o nounset
+  set +o errexit
+}