Skip to content

Commit

Permalink
Merge pull request #45 from ilrudie/master
Browse files Browse the repository at this point in the history 
Addressing request to extend testing options for functional-test automation
  • Loading branch information
@phenixblue
phenixblue committed Sep 30, 2020
2 parents d2d0fc2 + 9d3517a commit 2b3b859
Show file tree
Hide file tree
Showing 2 changed files with 171 additions and 97 deletions.
179 changes: 120 additions & 59 deletions hack/run-functional-tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
################################################################################

RUN_TYPE="${1}"
TEST_RESOURCE_TYPE="${2}"
TEST_TYPE="${3}"
TEST_RESOURCE_KIND="${2}"
TEST_RESOURCE_DESIRED="${3}"
TEST_NAMESPACE="${4}"
TESTS_MANIFEST="testing/functional-tests.yaml"

Expand All @@ -50,7 +50,7 @@ help_message() {
# **********************************************
check_arguments() {

if [ "${RUN_TYPE}" == "" ] || [ "${TEST_RESOURCE_TYPE}" == "" ] || [ "${TEST_TYPE}" == "" ] || [ "${TEST_NAMESPACE}" == "" ]; then
if [ "${RUN_TYPE}" == "" ] || [ "${TEST_RESOURCE_KIND}" == "" ] || [ "${TEST_RESOURCE_DESIRED}" == "" ] || [ "${TEST_NAMESPACE}" == "" ]; then

help_message
exit 1
Expand All @@ -60,7 +60,7 @@ check_arguments() {
help_message
exit 1

elif [ "${TEST_TYPE}" != "pass" ] && [ "${TEST_TYPE}" != "fail" ] && [ "${TEST_TYPE}" != "all" ]; then
elif [ "${TEST_RESOURCE_DESIRED}" != "pass" ] && [ "${TEST_RESOURCE_DESIRED}" != "fail" ] && [ "${TEST_RESOURCE_DESIRED}" != "all" ]; then

help_message
exit 1
Expand All @@ -73,107 +73,168 @@ check_arguments() {
# Run tests/cleanup
# **********************************************
run_resource_tests() {

# define local action from first argument
local action="${1}"

local action="${1}"
local resource="${2}"
local test_type="${3}"
local manifest_list=$(yq read -P "${TESTS_MANIFEST}" "resources.[name==${resource}].tests.${test_type}" | sed 's/^-[ ]*//')
# define local index from second argument
local resource_index="${2}"

if [ "${manifest_list}" == "" ]; then
# grab local resource from ${TESTS_MANIFEST}
local resource

resource=$(yq read "${TESTS_MANIFEST}" "resources.[${resource_index}].kind")

echo "[WARN] No \"${test_type}\" tests for \"${resource}\". Skipping..."
echo "============================================================================"
# grab local test type from ${TESTS_MANIFEST}
local test_type

test_type=$(yq read "${TESTS_MANIFEST}" "resources.[${resource_index}].desired")

# grab local list of test manifests to use
local manifest_list

manifest_list=$(yq read -P "${TESTS_MANIFEST}" "resources.[${resource_index}].manifests" | sed 's/^-[ ]*//')

# grab local user_script specified for pre/post/between running
local user_script

user_script=$(yq read -P "${TESTS_MANIFEST}" "resources.[${resource_index}].script")

# full path to the user specified script associate with this stanza in the manifest
local user_script_path="testing/${resource}/scripts/${user_script}"

if [ "${manifest_list}" == "" ]; then

echo "[WARN] No \"${test_type}\" tests for \"${resource}\". Skipping..."
echo "============================================================================"

else

echo "[INFO] **** Running \"${test_type}\" tests for \"${resource}\" ****"
echo "============================================================================"

else
# check to see if the user specified a script to be associated with this stanza
# only run script for apply actions
if [[ "${user_script}" != "" ]] && [[ "${action}" == "apply" ]]; then

# if they did specify a script run it with the setup argument and pass the namespace in
"${user_script_path}" "setup" "${TEST_NAMESPACE}"

echo "[INFO] **** Running \"${test_type}\" tests for \"${resource}\" ****"
echo "============================================================================"

for testfile in ${manifest_list}; do
fi

for testfile in ${manifest_list}; do

local test_file_path="testing/${resource}/${testfile}"
local test_file_path="testing/${resource}/${testfile}"

if [ -f "${test_file_path}" ]; then
if [ -f "${test_file_path}" ]; then

echo "[INFO] ${action}: \"${testfile}\""
echo "[INFO] ${action}: \"${testfile}\""

if [ "${action}" == "delete" ]; then

# kubectl doesn't like double quotes here.
# disable checking for double quotes around variables.
# shellcheck disable=SC2086
kubectl ${action} -f "${test_file_path}" -n ${TEST_NAMESPACE} --ignore-not-found

else

# kubectl doesn't like double quotes here.
# disable checking for double quotes around variables
# shellcheck disable=SC2086
kubectl ${action} -f "${test_file_path}" -n ${TEST_NAMESPACE}
local exit_code=$?

if [ "${action}" == "apply" ]; then
fi

if [ "${test_type}" == "pass" ] && [ ${exit_code} -ne 0 ]; then
local exit_code=$?

echo "[ERROR] Test did not pass. Exiting..."
exit 1
if [ "${action}" == "apply" ]; then

elif [ "${test_type}" == "fail" ] && [ ${exit_code} -ne 1 ]; then
if [ "${test_type}" == "pass" ] && [ ${exit_code} -ne 0 ]; then

echo "[ERROR] Test did not pass. Exiting..."
exit 1
echo "[ERROR] Test did not pass. Exiting..."
exit 1

else
elif [ "${test_type}" == "fail" ] && [ ${exit_code} -ne 1 ]; then

echo "[INFO] Test Passed"
echo "[ERROR] Test did not pass. Exiting..."
exit 1

fi
else

fi
echo "[INFO] Test Passed"

else
fi

echo "[WARN] File \"${test_file_path}\" not found. Skipping..."
fi

fi
else

echo "============================================================================"

done
echo "[WARN] File \"${test_file_path}\" not found. Skipping..."

fi
fi

}
# check to see if the user specified a script to be associated with this stanza
# only run script for apply actions
if [[ "${user_script}" != "" ]] && [[ "${action}" == "apply" ]]; then

# **********************************************
# Determine test scope
# **********************************************
scope_and_run_tests() {
# if they did specify a script run it with the between argument and pass the namespace in
"${user_script_path}" "between" "${TEST_NAMESPACE}"

local action="${1}"
fi

if [ "${TEST_RESOURCE_TYPE}" == "all" ]; then
echo "============================================================================"

done

resources=$(yq read "${TESTS_MANIFEST}" 'resources.[*].name')
# check to see if the user specified a script to be associated with this stanza
# only run script for apply actions
if [[ "${user_script}" != "" ]] && [[ "${action}" == "apply" ]]; then

for resource in ${resources}; do
# if they did specify a script run it with the teardown argument and pass the namespace in
"${user_script_path}" "teardown" "${TEST_NAMESPACE}"

if [ "${TEST_TYPE}" == "all" ]; then
echo "============================================================================"

run_resource_tests "${action}" "${resource}" "pass"
run_resource_tests "${action}" "${resource}" "fail"
fi

else
fi

run_resource_tests "${action}" "${resource}" "${TEST_TYPE}"
}

fi
# **********************************************
# Determine test scope
# **********************************************
scope_and_run_tests() {

done
# create identifiable local variable for argument 1, the action to perform
local action="${1}"

else
# size the array of resources
local resource_array_length
resource_array_length=$(yq read -l "${TESTS_MANIFEST}" 'resources')

if [ "${TEST_TYPE}" == "all" ]; then

run_resource_tests "${action}" "${resource}" "pass"
run_resource_tests "${action}" "${resource}" "fail"
# loop through all resources in the supplied manifest
# determine which indicies meet the supplied criteria in ${TEST_RESOURCE_KIND} and ${TEST_RESOURCE_DESIRED}
for ((i = 0 ; i < resource_array_length ; i++)); do

else
# check if we're doing all resources or if the resource kind at $i matches the requested kind
# double brackets are technically correct; the BEST kind of correct!
if [[ "${TEST_RESOURCE_KIND}" == "all" ]] || [[ "${TEST_RESOURCE_KIND}" == "$(yq read "${TESTS_MANIFEST}" "resources.[${i}].kind")" ]]; then

run_resource_tests "${action}" "${resource}" "${TEST_TYPE}"
# check if we're doing all desired results or if the resoured desired result at $i matches the requested desired result
if [[ "${TEST_RESOURCE_DESIRED}" == "all" ]] || [[ "${TEST_RESOURCE_DESIRED}" == "$(yq read "${TESTS_MANIFEST}" "resources.[${i}].desired")" ]]; then

run_resource_tests "${action}" "${i}"
fi

fi

fi
done

}

################################################################################
Expand Down
89 changes: 51 additions & 38 deletions testing/functional-tests.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,55 @@
resources:
- name: deployments
tests:
pass:
- test-deploy01.yaml
- test-deploy03.yaml
- test-deploy04.yaml
- test-deploy05.yaml
- test-deploy06.yaml
- test-deploy07.yaml
- test-deploy08.yaml
- test-deploy09.yaml
- test-deploy10.yaml
- test-deploy11.yaml
fail:
- test-deploy02.yaml
- name: pdbs
tests:
pass:
- test-pdb02.yaml
- test-pdb05.yaml
fail:
- test-pdb01.yaml
- test-pdb03.yaml
- test-pdb04.yaml
- test-pdb06.yaml
- name: pods
tests:
pass:
- test-pod02.yaml
- test-pod03.yaml
- test-pod04.yaml
- test-pod05.yaml
- test-pod06.yaml
fail:
- test-pod01.yaml
- name: services
tests:
pass:
- kind: deployments
desired: pass
script:
manifests:
- test-deploy01.yaml
- test-deploy03.yaml
- test-deploy04.yaml
- test-deploy05.yaml
- test-deploy06.yaml
- test-deploy07.yaml
- test-deploy08.yaml
- test-deploy09.yaml
- test-deploy10.yaml
- test-deploy11.yaml
- kind: deployments
desired: fail
script:
manifests:
- test-deploy02.yaml
- kind: pdbs
desired: pass
script:
manifests:
- test-pdb02.yaml
- test-pdb05.yaml
- kind: pdbs
desired: fail
script:
manifests:
- test-pdb01.yaml
- test-pdb03.yaml
- test-pdb04.yaml
- test-pdb06.yaml
- kind: pods
desired: pass
script:
manifests:
- test-pod02.yaml
- test-pod03.yaml
- test-pod04.yaml
- test-pod05.yaml
- test-pod06.yaml
- kind: pods
desired: fail
script:
manifests:
- test-pod01.yaml
- kind: services
desired: pass
script:
manifests:
- test-svc01.yaml
- test-svc02.yaml
- test-svc03.yaml
Expand Down

0 comments on commit 2b3b859

Please sign in to comment.