Add tests for helm completion (helm#6115)

This commit adds a framework and tests for helm command-line
completion.

To cover different shells (bash 4, bash 3, zsh) we use docker.

For MacOS, the tests are run locally if the host is MacOS and has
the right setup (bash completion installed, or zsh installed).

Dynamic-completion (e.g., helm status <TAB>) is not yet tested, but has
been considered in the design of the framework.

To run the tests:
  make test-completion

Although the framework supports marking tests as known failures,
this commit commented out the failing tests to allow for a clean
output for this proposal.  They can be uncommented later on.

A new directory is added: scripts/completion-tests.
It contains the required scripts for the completion tests.

scripts/completion-tests/completionTests.sh:
   Helm-specific completion tests.
   This script should evolve as new tests are added and new
   completion features implemented.

scripts/completion-tests/test-completion.sh:
   This script repeatedly runs the completion-tests in different
   environments using docker.
   This script would need changes if a new environment needs to
   be added.

scripts/completion-tests/lib/completionTests-base.sh:
   The magic. It is this script that allows completion testing.
   This script is not expected to change in day-to-day helm
   evolution.

Signed-off-by: Marc Khouzam <marc.khouzam@ville.montreal.qc.ca>

Makefile

@@ -81,6 +81,12 @@ test-style: vendor $(GOLANGCI_LINT)
 	$(GOLANGCI_LINT) run
 	@scripts/validate-license.sh
 
+.PHONY: test-completion
+test-completion: TARGETS = linux/amd64
+test-completion: build build-cross
+test-completion:
+	scripts/completion-tests/test-completion.sh
+
 .PHONY: verify-docs
 verify-docs: build
 	@scripts/verify-docs.sh

scripts/completion-tests/completionTests.sh

@@ -0,0 +1,64 @@
+#!bash
+#
+# Copyright (C) 2019 Ville de Montreal
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script tests different scenarios of completion.  The tests can be
+# run by sourcing this file from a bash shell or a zsh shell.
+
+source /tmp/completion-tests/lib/completionTests-base.sh
+
+# Don't use the new source <() form as it does not work with bash v3
+source /dev/stdin <<- EOF
+   $(helm completion $SHELL_TYPE)
+EOF
+
+# No need to test every command, as completion is handled
+# automatically by Cobra.
+# We focus on some smoke tests for the Cobra-handled completion
+# and also on code specific to this project.
+
+# Basic first level commands (static completion)
+_completionTests_verifyCompletion "helm stat" "status"
+_completionTests_verifyCompletion "helm status" "status"
+_completionTests_verifyCompletion "helm lis" "list"
+_completionTests_verifyCompletion "helm r" "registry repo rollback"
+_completionTests_verifyCompletion "helm re" "registry repo"
+
+# Basic second level commands (static completion)
+_completionTests_verifyCompletion "helm get " "hooks manifest values"
+_completionTests_verifyCompletion "helm get h" "hooks"
+_completionTests_verifyCompletion "helm completion " "bash zsh"
+_completionTests_verifyCompletion "helm completion z" "zsh"
+
+# Completion of flags
+#_completionTests_verifyCompletion ZFAIL "helm --kube-con" "--kube-context= --kube-context"
+#_completionTests_verifyCompletion ZFAIL "helm --kubecon" "--kubeconfig= --kubeconfig"
+#_completionTests_verifyCompletion ZFAIL "helm --name" "--namespace= --namespace"
+_completionTests_verifyCompletion "helm -v" "-v"
+#_completionTests_verifyCompletion ZFAIL "helm --v" "--v= --vmodule= --v --vmodule"
+
+# Completion of commands while using flags
+_completionTests_verifyCompletion "helm --kube-context prod sta" "status"
+_completionTests_verifyCompletion "helm --namespace mynamespace get h" "hooks"
+#_completionTests_verifyCompletion KFAIL "helm -v get " "hooks manifest values"
+#_completionTests_verifyCompletion ZFAIL "helm --kubeconfig=/tmp/config lis" "list"
+#_completionTests_verifyCompletion ZFAIL "helm ---namespace mynamespace get " "hooks manifest values"
+#_completionTests_verifyCompletion ZFAIL "helm get --name" "--namespace= --namespace"
+#_completionTests_verifyCompletion ZFAIL "helm get hooks --kubec" "--kubeconfig= --kubeconfig"
+
+# Alias completion
+# Does not work.
+#_completionTests_verifyCompletion KFAIL "helm ls" "ls"
+#_completionTests_verifyCompletion KFAIL "helm dependenci" "dependencies"

scripts/completion-tests/lib/completionTests-base.sh

@@ -0,0 +1,152 @@
+#!bash
+#
+# Copyright (C) 2019 Ville de Montreal
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script allows to run completion tests for the bash shell.
+# It also supports zsh completion tests, when zsh is used in bash-completion
+# compatibility mode.
+#
+# To use this script one should create a test script which will:
+# 1- source this script
+# 2- source the completion script to be tested
+# 3- call repeatedly the _completionTests_verifyCompletion() function passing it
+#    the command line to be completed followed by the expected completion.
+#
+# For example, the test script can look like this:
+#
+# #!bash
+# # source completionTests-base.sh
+# # source helmCompletionScript.${SHELL_TYPE}
+# # _completionTests_verifyCompletion "helm stat" "status"
+#
+
+# Global variable to keep track of if a test has failed.
+_completionTests_TEST_FAILED=0
+
+# Run completion and indicate success or failure.
+#    $1 is the command line that should be completed
+#    $2 is the expected result of the completion
+# If $1 = KFAIL indicates a Known failure
+#    $1 = BFAIL indicates a Known failure only for bash
+#    $1 = ZFAIL indicates a Known failure only for zsh
+_completionTests_verifyCompletion() {
+   local expectedFailure="NO"
+   case $1 in
+   [K,B,Z]FAIL)
+      expectedFailure=$1
+      shift
+      ;;
+   esac
+
+   local cmdLine=$1
+   local expected=$2
+
+   result=$(_completionTests_complete "${cmdLine}")
+
+   if [ $expectedFailure = "KFAIL" ] ||
+           ([ $expectedFailure = "BFAIL" ] && [ $SHELL_TYPE = "bash" ]) ||
+           ([ $expectedFailure = "ZFAIL" ] && [ $SHELL_TYPE = "zsh" ]); then
+      if [ "$result" = "$expected" ]; then
+         _completionTests_TEST_FAILED=1
+         echo "UNEXPECTED SUCCESS: \"$cmdLine\" completes to \"$result\""
+      else
+         echo "$expectedFailure: \"$cmdLine\" should complete to \"$expected\" but we got \"$result\""
+      fi
+   elif [ "$result" = "$expected" ]; then
+      echo "SUCCESS: \"$cmdLine\" completes to \"$result\""
+   else
+      _completionTests_TEST_FAILED=1
+      echo "FAIL: \"$cmdLine\" should complete to \"$expected\" but we got \"$result\""
+   fi
+
+   # Return the global result each time.  This allows for the very last call to
+   # this method to return the correct success or failure code for the entire script
+   return $_completionTests_TEST_FAILED
+}
+
+# Find the completion function associated with the binary.
+# $1 is the name of the binary for which completion was triggered.
+_completionTests_findCompletionFunction() {
+    local out=($(complete -p $1))
+    local returnNext=0
+    for i in ${out[@]}; do
+       if [ $returnNext -eq 1 ]; then
+          echo "$i"
+          return
+       fi
+       [ "$i" = "-F" ] && returnNext=1
+    done
+}
+
+_completionTests_complete() {
+   local cmdLine=$1
+
+   # Set the bash completion variables which are
+   # used for both bash and zsh completion
+   COMP_LINE=${cmdLine}
+   COMP_POINT=${#COMP_LINE}
+   COMP_TYPE=9 # 9 is TAB
+   COMP_KEY=9  # 9 is TAB
+   COMP_WORDS=($(echo ${cmdLine}))
+
+   COMP_CWORD=$((${#COMP_WORDS[@]}-1))
+   # We must check for a space as the last character which will tell us
+   # that the previous word is complete and the cursor is on the next word.
+   [ "${cmdLine: -1}" = " " ] && COMP_CWORD=${#COMP_WORDS[@]}
+
+   # Call the completion function associated with the binary being called.
+   eval $(_completionTests_findCompletionFunction ${COMP_WORDS[0]})
+
+   # Return the result of the completion.
+   echo "${COMPREPLY[@]}"
+}
+
+# compopt, which is only available for bash 4, I believe,
+# prints an error when it is being called outside of real shell
+# completion.  Since it doesn't work anyway in our case, let's
+# disable it to avoid the error printouts.
+# Impacts are limited to completion of flags and even then
+# for zsh and bash 3, it is not even available.
+compopt() {
+   :
+}
+
+# Start of script
+SHELL_TYPE=bash
+if [ ! -z "$BASH_VERSION" ];then
+   echo "===================================================="
+   echo "Running completions tests on $(uname) with bash $BASH_VERSION"
+   echo "===================================================="
+
+   bashCompletionScript="/usr/share/bash-completion/bash_completion"
+   if [ $(uname) = "Darwin" ]; then
+      bashCompletionScript="/usr/local/etc/bash_completion"
+   fi
+
+   source ${bashCompletionScript}
+else
+   SHELL_TYPE=zsh
+
+   echo "===================================================="
+   echo "Running completions tests on $(uname) with zsh $ZSH_VERSION"
+   echo "===================================================="
+   autoload -Uz compinit
+   compinit
+   # When zsh calls real completion, it sets some options and emulates sh.
+   # We need to do the same.
+   emulate -L sh
+   setopt kshglob noshglob braceexpand
+fi

scripts/completion-tests/test-completion.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2019 Ville de Montreal
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script runs completion tests in different environments and different shells.
+
+# Fail as soon as there is an error
+set -e
+
+SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+BINARY_NAME=helm
+BINARY_PATH=${SCRIPT_DIR}/../../_dist/linux-amd64
+
+if [ -z $(which docker) ]; then
+  echo "Missing 'docker' client which is required for these tests";
+  exit 2;
+fi
+
+COMP_DIR=/tmp/completion-tests
+COMP_SCRIPT_NAME=completionTests.sh
+COMP_SCRIPT=${COMP_DIR}/${COMP_SCRIPT_NAME}
+
+mkdir -p ${COMP_DIR}/lib
+cp ${SCRIPT_DIR}/${COMP_SCRIPT_NAME} ${COMP_DIR}
+cp ${SCRIPT_DIR}/lib/completionTests-base.sh ${COMP_DIR}/lib
+cp ${BINARY_PATH}/${BINARY_NAME} ${COMP_DIR}
+
+########################################
+# Bash 4 completion tests
+########################################
+BASH4_IMAGE=completion-bash4
+
+echo;echo;
+docker build -t ${BASH4_IMAGE} - <<- EOF
+   FROM bash:4.4
+   RUN apk update && apk add bash-completion
+EOF
+docker run --rm \
+           -v ${COMP_DIR}:${COMP_DIR} -v ${COMP_DIR}/${BINARY_NAME}:/bin/${BINARY_NAME} \
+           ${BASH4_IMAGE} bash -c "source ${COMP_SCRIPT}"
+
+########################################
+# Bash 3.2 completion tests
+########################################
+# We choose version 3.2 because we want some Bash 3 version and 3.2
+# is the version by default on MacOS.  So testing that version
+# gives us a bit of coverage for MacOS.
+BASH3_IMAGE=completion-bash3
+
+echo;echo;
+docker build -t ${BASH3_IMAGE} - <<- EOF
+   FROM bash:3.2
+   # For bash 3.2, the bash-completion package required is version 1.3
+   RUN mkdir /usr/share/bash-completion && \
+       wget -qO - https://github.com/scop/bash-completion/archive/1.3.tar.gz | \
+            tar xvz -C /usr/share/bash-completion --strip-components 1 bash-completion-1.3/bash_completion
+EOF
+docker run --rm \
+           -v ${COMP_DIR}:${COMP_DIR} -v ${COMP_DIR}/${BINARY_NAME}:/bin/${BINARY_NAME} \
+           -e BASH_COMPLETION=/usr/share/bash-completion \
+           ${BASH3_IMAGE} bash -c "source ${COMP_SCRIPT}"
+
+########################################
+# Zsh completion tests
+########################################
+ZSH_IMAGE=completion-zsh
+
+echo;echo;
+docker build -t ${ZSH_IMAGE} - <<- EOF
+   FROM zshusers/zsh:5.7
+EOF
+docker run --rm \
+           -v ${COMP_DIR}:${COMP_DIR} -v ${COMP_DIR}/${BINARY_NAME}:/bin/${BINARY_NAME} \
+           ${ZSH_IMAGE} zsh -c "source ${COMP_SCRIPT}"
+
+########################################
+# MacOS completion tests
+########################################
+# Since we can't use Docker to test MacOS,
+# we run the MacOS tests locally when possible.
+if [ "$(uname)" == "Darwin" ]; then
+   # Make sure that for the local tests, the tests will find the newly
+   # built binary.  If for some reason the binary to test is not present
+   # the tests may use the default binary installed on localhost and we
+   # won't be testing the right thing.  So we check here.
+   if [ $(PATH=$(pwd)/bin:$PATH which ${BINARY_NAME}) != $(pwd)/bin/${BINARY_NAME} ]; then
+      echo "Cannot find ${BINARY_NAME} under $(pwd)/bin/${BINARY_NAME} although it is what we need to test."
+      exit 1
+   fi
+
+   if which bash>/dev/null && [ -f /usr/local/etc/bash_completion ]; then
+      echo;echo;
+      echo "Completion tests for bash running locally"
+      PATH=$(pwd)/bin:$PATH bash -c "source ${COMP_SCRIPT}"
+   fi
+
+   if which zsh>/dev/null; then
+      echo;echo;
+      echo "Completion tests for zsh running locally"
+      PATH=$(pwd)/bin:$PATH zsh -c "source ${COMP_SCRIPT}"
+   fi
+fi