Skip to content

Commit

Permalink
feat: add helm chart tests for gateway api routes (#345)
Browse files Browse the repository at this point in the history
Signed-off-by: Lenin Mehedy <lenin.mehedy@swirldslabs.com>
Co-authored-by: Nathan Klick <nathan@swirldslabs.com>
  • Loading branch information
leninmehedy and nathanklick committed Sep 15, 2023
1 parent af2a8b8 commit 3936a64
Show file tree
Hide file tree
Showing 11 changed files with 264 additions and 35 deletions.
18 changes: 15 additions & 3 deletions charts/hedera-network/templates/rbac/pod-monitor-role.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
kind: ClusterRole
metadata:
name: pod-monitoring-role
namespace: {{ .Values.namespace }}
rules:
- apiGroups: [ "" ]
resources:
Expand All @@ -16,15 +17,26 @@ rules:
- pods/exec
verbs:
- create
- apiGroups: [ "gateway.networking.k8s.io" ]
resources:
- gatewayclasses
- gateways
- httproutes
- tcproutes
verbs:
- get
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
kind: ClusterRoleBinding
metadata:
name: pod-monitoring-role-binding
namespace: {{ .Values.namespace }}
subjects:
- kind: ServiceAccount
name: pod-monitor
namespace: {{ .Values.namespace }}
roleRef:
kind: Role
kind: ClusterRole
name: pod-monitoring-role
apiGroup: rbac.authorization.k8s.io
3 changes: 2 additions & 1 deletion charts/hedera-network/templates/rbac/service-accounts.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: pod-monitor
name: pod-monitor
namespace: {{ .Values.namespace }}
19 changes: 19 additions & 0 deletions charts/hedera-network/tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,25 @@ you have the network deployed already.

## Development
- Use the `test_basic_deployment.bats` file as the template while creating new tests.
- In order to run and debug the tests inside the helm test container, do the following:

- Update `run` command section in `charts/hedera-network/template/tests/test-deployment.yaml` as below so that it keeps it running when we run `make helm-test`:
```
- "/bin/bash"
- "-c"
#- "/tests/run.sh"
- "while true;do echo sleeping for 60s; sleep 60;done" # keep the test container running so that we can debug issues
```
- once the `network-test` container is running, use another terminal to shell into it using command below
```
kubectl exec -it network-test -- bash
```
- Then you can run the test inside the container to debug
```
cd /tests && ./run.sh
```
- Once debug is done, you can exit and use Ctrl+C to terminate the helm-test process (you will need to delete the `network-test` container using `kubectl delete network-test`).
- If it looks all good, revert changes in `charts/hedera-network/template/tests/test-deployment.yaml`

## Run
- Run `git submodule update --init` in order to install [bats](https://github.com/bats-core) for tests.
Expand Down
65 changes: 60 additions & 5 deletions charts/hedera-network/tests/helper.sh
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,10 @@ function check_test_status() {

function get_config_val() {
local config_path=$1
ret=$(helm get values fst -a | tail -n +2 | niet "${config_path}" )
log_debug "Get config command: helm get values fst -a | yq '${config_path}'"
ret=$(helm get values fst -a | yq "${config_path}" )
echo "${ret}"
log_debug "${enable_config_path} => ${ret}"
log_debug "${config_path} => ${ret}"
}

function get_config_val_upper() {
Expand All @@ -93,6 +94,40 @@ function get_config_val_upper() {
echo "${config_val}"
}

function is_enabled_for_node() {
local node_name=$1
[[ -z "${node_name}" ]] && echo "ERROR: Node name is needed" && return "${EX_ERR}"

local config_path=$2
[[ -z "${config_path}" ]] && echo "ERROR: Config path is needed" && return "${EX_ERR}"

log_debug "Checking config '${config_path}' for node '${node_name}"

local default_config_path=".defaults${config_path}"
local node_config_path=".hedera.nodes[] | select(.name==\"${node_name}\") | ${config_path}"
local default_val=$(get_config_val_upper "${default_config_path}")
local node_val=$(get_config_val_upper "${node_config_path}")
log_debug "Default config: ${default_val}"
log_debug "Node config: ${node_val}"

if [ -z "${node_val}" ] || [ "${node_val}" = "FALSE" ]; then
echo "FALSE"
return
fi

if [ "${node_val}" = "TRUE" ]; then
echo "TRUE"
return
fi

if [ "${default_val}" = "TRUE" ]; then
echo "TRUE"
return
fi

echo "FALSE"
}

function get_sidecar_status() {
local pod=$1
local sidecar_name=$2
Expand All @@ -110,6 +145,7 @@ function is_sidecar_ready() {
[[ -z "${sidecar_name}" ]] && echo "ERROR: Sidecar name is needed (is_sidecar_ready)" && return "${EX_ERR}"

local sidecar_status=$(kubectl get pod "${pod}" -o jsonpath="{.status.containerStatuses[?(@.name=='${sidecar_name}')].ready}" | tr '[:lower:]' '[:upper:]')
[ -z "${sidecar_status}" ] && sidecar_status="FALSE"
log_debug "${sidecar_name} in pod ${pod} is ready: ${sidecar_status}"

[[ "${sidecar_status}" = "TRUE" ]] && return "${EX_OK}"
Expand Down Expand Up @@ -138,6 +174,8 @@ function is_pod_ready() {
[[ -z "${pod}" ]] && echo "ERROR: Pod name is needed (is_pod_ready)" && return "${EX_ERR}"

local pod_status=$(kubectl get pod "${pod}" -o jsonpath="{.status.conditions[?(@.type=='Ready')].status}" | tr '[:lower:]' '[:upper:]')
[ -z "${pod_status}" ] && pod_status="FALSE"

log_debug "Pod '${pod}' is ready: ${pod_status}"

[[ "${pod_status}" = "TRUE" ]] && return "${EX_OK}"
Expand All @@ -152,12 +190,13 @@ function get_pod_label() {
[[ -z "${pod}" ]] && echo "ERROR: Label name is needed" && return "${EX_ERR}"


log_debug "Checking for pod ${pod}(timeout 300s)..."
if [ ! $(kubectl wait --for=condition=Initialized pods "${pod}" --timeout 300s) ]; then
log_debug "Checking for pod '${pod}'(timeout 300s)..."
$(kubectl wait --for=condition=Initialized pods "${pod}" --timeout 300s) > /dev/null 2>&1
if [ $? = 1 ]; then
log_debug "ERROR: Pod ${pod} is not available" && return "${EX_ERR}"
fi

log_debug "Checking label ${label} for pod ${pod}"
log_debug "Checking label '${label}' for pod '${pod}'"
local escaped_label="${label//./\\.}"
local label_val=$(kubectl get pod "${pod}" -o jsonpath="{.metadata.labels.${escaped_label}}" | xargs)
log_debug "Pod '${pod}' label '${label}': ${label_val}"
Expand All @@ -174,3 +213,19 @@ function get_pod_by_label() {
local pod_name=$(kubectl get pods -l "${label}" -o jsonpath="{.items[0].metadata.name}")
echo "${pod_name}"
}

function is_route_accepted() {
local route_type=$1
[[ -z "${route_type}" ]] && echo "ERROR: Route type is needed" && return "${EX_ERR}"

local route_name=$2
[[ -z "${route_name}" ]] && echo "ERROR: Route name is needed" && return "${EX_ERR}"

local route_status=$(kubectl get "${route_type}" "${route_name}" -o jsonpath="{.status.parents[*].conditions[?(@.type=='Accepted')].status}" | tr '[:lower:]' '[:upper:]')
[ -z "${route_status}" ] && route_status="FALSE"

log_debug "${route_type} '${route_name}' is accepted: ${route_status}"

[[ "${route_status}" = "TRUE" ]] && return "${EX_OK}"
return "${EX_ERR}"
}
121 changes: 121 additions & 0 deletions charts/hedera-network/tests/test_gateway_api_deployment.bats
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
# bats file_tags=deployment-test
setup() {
source "$(dirname "${BATS_TEST_FILENAME}")/env.sh"
source "${TESTS_DIR}/load.sh"
}

@test "Check Envoy Proxy GRPC-Web routes" {
log_debug "----------------------------------------------------------------------------"
log_debug "TEST: Checking Envoy Proxy GRPC-Web Route"
log_debug "----------------------------------------------------------------------------"

local resp="$(get_pod_list network-node)"
local pods=(${resp}) # convert into an array
log_debug "Network nodes: ${pods[*]}"

local test_status="${FAIL}"
local status_val="${EX_ERR}"
if [[ "${#pods[@]}" -gt 0 ]]; then
test_status="${PASS}"
for pod in "${pods[@]}"; do
log_debug ""
local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name")
[[ -z "${node_name}" ]] && test_status="${FAIL}" && break

local route_name="envoy-grpc-web-route-${node_name}"
local is_enabled=$(is_enabled_for_node "${node_name}" ".envoyProxy.enable")
if [ "${is_enabled}" = "TRUE" ]; then
log_debug "EnvoyProxy enabled for node '${node_name}'"
log_debug "Checking Envoy proxy httproute '${route_name}'"
is_route_accepted "httproute" "${route_name}" || test_status="${FAIL}"
[ "${test_status}" = "FAIL" ] && break
else
log_debug "EnvoyProxy is not enabled for node '${node_name}'. Skipped route check for '${route_name}'."
fi
done
fi

log_debug ""
log_debug "[${test_status}] Envoy Proxy GRPC-Web Route Check"
log_debug ""

# assert success
[[ "${test_status}" = "${PASS}" ]]
}

@test "Check HAProxy GRPC routes" {
log_debug "----------------------------------------------------------------------------"
log_debug "TEST: Checking HAProxy GRPC Route"
log_debug "----------------------------------------------------------------------------"

local resp="$(get_pod_list network-node)"
local pods=(${resp}) # convert into an array
log_debug "Network nodes: ${pods[*]}"

local test_status="${FAIL}"
local status_val="${EX_ERR}"
if [[ "${#pods[@]}" -gt 0 ]]; then
test_status="${PASS}"
for pod in "${pods[@]}"; do
log_debug ""
local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name")
[[ -z "${node_name}" ]] && test_status="${FAIL}" && break

local route_name="haproxy-grpc-route-${node_name}"
local is_enabled=$(is_enabled_for_node "${node_name}" ".haproxy.enable")
if [ "${is_enabled}" = "TRUE" ]; then
log_debug "HAProxy enabled for node '${node_name}'"
log_debug "Checking HAProxy GRPC route '${route_name}'"
is_route_accepted "tcproute" "${route_name}" || test_status="${FAIL}"
[ "${test_status}" = "FAIL" ] && break
else
log_debug "HAProxy is not enabled for node '${node_name}'. Skipped route check for '${route_name}'."
fi
done
fi

log_debug ""
log_debug "[${test_status}] HAProxy GRPC Route Check"
log_debug ""

# assert success
[[ "${test_status}" = "${PASS}" ]]
}

@test "Check Network Node GRPC routes" {
log_debug "----------------------------------------------------------------------------"
log_debug "TEST: Checking Network Node GRPC Route"
log_debug "----------------------------------------------------------------------------"

local resp="$(get_pod_list network-node)"
local pods=(${resp}) # convert into an array
log_debug "Network nodes: ${pods[*]}"

local test_status="${FAIL}"
local status_val="${EX_ERR}"
if [[ "${#pods[@]}" -gt 0 ]]; then
test_status="${PASS}"
for pod in "${pods[@]}"; do
log_debug ""
local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name")
[[ -z "${node_name}" ]] && test_status="${FAIL}" && break

local route_name="node-grpc-route-${node_name}"
local is_enabled=$(is_enabled_for_node "${node_name}" ".haproxy.enable")
if [ "${is_enabled}" = "FALSE" ]; then
log_debug "Checking Node GRPC route '${route_name}'"
is_route_accepted "tcproute" "${route_name}" || test_status="${FAIL}"
[ "${test_status}" = "FAIL" ] && break
else
log_debug "HAProxy enabled for node '${node_name}'. Skipped route check for '${route_name}'."
fi
done
fi

log_debug ""
log_debug "[${test_status}] Node GRPC Route Check"
log_debug ""

# assert success
[[ "${test_status}" = "${PASS}" ]]
}
33 changes: 22 additions & 11 deletions charts/hedera-network/tests/test_proxy_deployment.bats
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,23 @@ setup() {
local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name")
[[ -z "${node_name}" ]] && test_status="${FAIL}" && break

log_debug "Checking HAProxy for network-node '${node_name}'"
local haproxy_pod=$(get_pod_by_label "app=haproxy-${node_name},fullstack.hedera.com/type=haproxy")

log_debug "Checking HAProxy pod ${haproxy_pod}"
is_pod_ready "${haproxy_pod}" || test_status="${FAIL}"
local is_enabled=$(is_enabled_for_node "${node_name}" ".haproxy.enable")
if [ "${is_enabled}" = "TRUE" ]; then
log_debug "HAProxy is enabled for node '${node_name}'"
log_debug "Checking HAProxy for network-node '${node_name}'"
local haproxy_pod=$(get_pod_by_label "app=haproxy-${node_name},fullstack.hedera.com/type=haproxy")
log_debug "Checking HAProxy pod ${haproxy_pod}"
is_pod_ready "${haproxy_pod}" || test_status="${FAIL}"
else
log_debug "HAProxy is not enabled for node '${node_name}'. Skipping check"
fi

[ "${test_status}" = "FAIL" ] && break
done
fi

log_debug ""
log_debug "[${test_status}] HAProxy is running"
log_debug "[${test_status}] HAProxy Check"
log_debug ""

# assert success
Expand All @@ -58,18 +63,24 @@ setup() {
local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name")
[[ -z "${node_name}" ]] && test_status="${FAIL}" && break

log_debug "Checking Envoy proxy for network-node '${node_name}'"
local envoy_proxy_pod=$(get_pod_by_label "app=envoy-proxy-${node_name},fullstack.hedera.com/type=envoy-proxy")
local is_enabled=$(is_enabled_for_node "${node_name}" ".envoyProxy.enable")
if [ "${is_enabled}" = "TRUE" ]; then
log_debug "EnvoyProxy is enabled for node '${node_name}'"
log_debug "Checking Envoy proxy for network-node '${node_name}'"
local envoy_proxy_pod=$(get_pod_by_label "app=envoy-proxy-${node_name},fullstack.hedera.com/type=envoy-proxy")

log_debug "Checking Envoy Proxy pod ${envoy_proxy_pod}"
is_pod_ready "${envoy_proxy_pod}" || test_status="${FAIL}"
log_debug "Checking Envoy Proxy pod ${envoy_proxy_pod}"
is_pod_ready "${envoy_proxy_pod}" || test_status="${FAIL}"
else
log_debug "EnvoyProxy is not enabled for node '${node_name}'. Skipping check"
fi

[ "${test_status}" = "FAIL" ] && break
done
fi

log_debug ""
log_debug "[${test_status}] Envoy Proxy is running"
log_debug "[${test_status}] Envoy Proxy Check"
log_debug ""

# assert success
Expand Down
Loading

0 comments on commit 3936a64

Please sign in to comment.