From f312195e9685a9c3ed9f561210c92ac48d7b0a17 Mon Sep 17 00:00:00 2001 From: ryanohnemus Date: Thu, 11 Jan 2024 07:01:32 -0600 Subject: [PATCH 1/4] Add kubernetes-plugins ci initial test Signed-off-by: ryanohnemus --- tests/kubernetes-plugins/basic.bats | 102 ++++++++++++++++++ .../resources/elasticsearch-basic.yaml | 13 +++ .../resources/fluentbit-basic.yaml | 56 ++++++++++ 3 files changed, 171 insertions(+) create mode 100644 tests/kubernetes-plugins/basic.bats create mode 100644 tests/kubernetes-plugins/resources/elasticsearch-basic.yaml create mode 100644 tests/kubernetes-plugins/resources/fluentbit-basic.yaml diff --git a/tests/kubernetes-plugins/basic.bats b/tests/kubernetes-plugins/basic.bats new file mode 100644 index 0000000..a5b7604 --- /dev/null +++ b/tests/kubernetes-plugins/basic.bats @@ -0,0 +1,102 @@ +#!/usr/bin/env bats + +load "$HELPERS_ROOT/test-helpers.bash" + +ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG + +load "$BATS_DETIK_ROOT/utils.bash" +load "$BATS_DETIK_ROOT/linter.bash" +load "$BATS_DETIK_ROOT/detik.bash" +load "$BATS_SUPPORT_ROOT/load.bash" +load "$BATS_ASSERT_ROOT/load.bash" +load "$BATS_FILE_ROOT/load.bash" + +setup_file() { + echo "recreating namespace $TEST_NAMESPACE" + run kubectl delete namespace "$TEST_NAMESPACE" + run kubectl create namespace "$TEST_NAMESPACE" + # HELM_VALUES_EXTRA_FILE is a default file containing global helm + # options that can be optionally applied on helm install/upgrade + # by the test. This will fall back to $TEST_ROOT/defaults/values.yaml.tpl + # if not passed. + if [ -e "${HELM_VALUES_EXTRA_FILE}" ]; then + envsubst < "${HELM_VALUES_EXTRA_FILE}" > "${HELM_VALUES_EXTRA_FILE%.*}" + export HELM_VALUES_EXTRA_FILE="${HELM_VALUES_EXTRA_FILE%.*}" + fi +} + +teardown_file() { + if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then + run kubectl delete namespace "$TEST_NAMESPACE" + rm -f ${HELM_VALUES_EXTRA_FILE} + fi +} + +# These are required for bats-detik +# shellcheck disable=SC2034 +DETIK_CLIENT_NAME="kubectl -n $TEST_NAMESPACE" +# shellcheck disable=SC2034 +DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" + + +@test "test fluent-bit reads k8s labels" { + helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" elasticsearch elastic/elasticsearch \ + --values ${BATS_TEST_DIRNAME}/resources/elasticsearch-basic.yaml \ + --set image=${ELASTICSEARCH_IMAGE_REPOSITORY} --set imageTag=${ELASTICSEARCH_IMAGE_TAG} \ + --values "$HELM_VALUES_EXTRA_FILE" \ + --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \ + --wait + + try "at most 15 times every 2s " \ + "to find 1 pods named 'elasticsearch-master-0' " \ + "with 'status' being 'running'" + + helm repo add elastic https://helm.elastic.co/ || helm repo add elastic https://helm.elastic.co + helm repo add fluent https://fluent.github.io/helm-charts/ || helm repo add fluent https://fluent.github.io/helm-charts + helm repo update --fail-on-repo-update-fail + + helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" fluent-bit fluent/fluent-bit \ + --values ${BATS_TEST_DIRNAME}/resources/fluentbit-basic.yaml \ + --set image.repository=${FLUENTBIT_IMAGE_REPOSITORY},image.tag=${FLUENTBIT_IMAGE_TAG} \ + --values "$HELM_VALUES_EXTRA_FILE" \ + --timeout "${HELM_FB_TIMEOUT:-5m0s}" \ + --wait + + try "at most 15 times every 2s " \ + "to find 1 pods named 'fluent-bit' " \ + "with 'status' being 'running'" + + # Wait for initial data to be reported + attempt=0 + while true; do + run kubectl exec -q -n "$TEST_NAMESPACE" elasticsearch-master-0 -- curl --insecure -s -w "%{http_code}" http://localhost:9200/fluentbit/_search/ -o /dev/null + if [[ "$output" != "200" ]]; then + if [ "$attempt" -lt 25 ]; then + attempt=$(( attempt + 1 )) + sleep 5 + else + fail "did not find any index results even after $attempt attempts" + fi + else + break + fi + done + + # Search for what we want + attempt=0 + while true; do + search="http://localhost:9200/fluentbit/_search/?q=kubernetes.namespace_name:*&size=1&filter_path=hits.hits._source.kubernetes.labels" + run kubectl exec -q -n "$TEST_NAMESPACE" elasticsearch-master-0 -- curl --insecure -s "$search" + + if [[ "$output" != *'{"kubernetes":{"labels":'* ]]; then + if [ "$attempt" -lt 25 ]; then + attempt=$(( attempt + 1 )) + sleep 5 + else + fail "did not find any kubernetes pod labels in output after $attempt attempts" + fi + else + break + fi + done +} \ No newline at end of file diff --git a/tests/kubernetes-plugins/resources/elasticsearch-basic.yaml b/tests/kubernetes-plugins/resources/elasticsearch-basic.yaml new file mode 100644 index 0000000..3de9b97 --- /dev/null +++ b/tests/kubernetes-plugins/resources/elasticsearch-basic.yaml @@ -0,0 +1,13 @@ +protocol: http +httpPort: 9200 +transportPort: 9300 +replicas: 1 +minimumMasterNodes: 1 +createCert: false +extraEnvs: + - name: xpack.security.enabled + value: "false" + - name: xpack.security.http.ssl.enabled + value: "false" + - name: xpack.security.transport.ssl.enabled + value: "false" diff --git a/tests/kubernetes-plugins/resources/fluentbit-basic.yaml b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml new file mode 100644 index 0000000..7800633 --- /dev/null +++ b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml @@ -0,0 +1,56 @@ +kind: Deployment +replicaCount: 1 +rbac: + create: true +extraVolumeMounts: +- mountPath: /var/log + name: varlog +- mountPath: /output + name: output +extraVolumes: +- name: varlog + hostPath: + path: /var/log +- name: output + emptyDir: + sizeLimit: 5Mi +config: + service: | + [SERVICE] + Flush 5 + Daemon Off + Log_Level debug + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + + inputs: | + [INPUT] + name tail + read_from_head true + path /var/log/containers/*.log + multiline.parser docker, cri + Tag kube.* + buffer_chunk_size 2M + buffer_max_size 2M + Exclude_Path /var/log/containers/fluent-bit* + + filters: | + [FILTER] + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc:443 + Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token + Kube_Tag_Prefix kube.var.log.containers. + Merge_Log Off + Annotations Off + Labels On + + outputs: | + [OUTPUT] + Name es + Match * + Host elasticsearch-master + Port 9200 + Index fluentbit \ No newline at end of file From fe2dc3052abcf9d359bd09add496feebab007201 Mon Sep 17 00:00:00 2001 From: ryanohnemus Date: Thu, 11 Jan 2024 16:54:11 -0600 Subject: [PATCH 2/4] kubernetes-plugins with stdout, elasticsearch out removed Signed-off-by: ryanohnemus --- tests/kubernetes-plugins/basic.bats | 55 +++++++------------ .../resources/elasticsearch-basic.yaml | 13 ----- .../resources/fluentbit-basic.yaml | 18 +++--- 3 files changed, 30 insertions(+), 56 deletions(-) delete mode 100644 tests/kubernetes-plugins/resources/elasticsearch-basic.yaml diff --git a/tests/kubernetes-plugins/basic.bats b/tests/kubernetes-plugins/basic.bats index a5b7604..3024015 100644 --- a/tests/kubernetes-plugins/basic.bats +++ b/tests/kubernetes-plugins/basic.bats @@ -2,7 +2,7 @@ load "$HELPERS_ROOT/test-helpers.bash" -ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG ELASTICSEARCH_IMAGE_REPOSITORY ELASTICSEARCH_IMAGE_TAG +ensure_variables_set BATS_SUPPORT_ROOT BATS_ASSERT_ROOT BATS_DETIK_ROOT BATS_FILE_ROOT TEST_NAMESPACE FLUENTBIT_IMAGE_REPOSITORY FLUENTBIT_IMAGE_TAG load "$BATS_DETIK_ROOT/utils.bash" load "$BATS_DETIK_ROOT/linter.bash" @@ -40,24 +40,12 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" @test "test fluent-bit reads k8s labels" { - helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" elasticsearch elastic/elasticsearch \ - --values ${BATS_TEST_DIRNAME}/resources/elasticsearch-basic.yaml \ - --set image=${ELASTICSEARCH_IMAGE_REPOSITORY} --set imageTag=${ELASTICSEARCH_IMAGE_TAG} \ - --values "$HELM_VALUES_EXTRA_FILE" \ - --timeout "${HELM_DEFAULT_TIMEOUT:-10m0s}" \ - --wait - - try "at most 15 times every 2s " \ - "to find 1 pods named 'elasticsearch-master-0' " \ - "with 'status' being 'running'" - - helm repo add elastic https://helm.elastic.co/ || helm repo add elastic https://helm.elastic.co helm repo add fluent https://fluent.github.io/helm-charts/ || helm repo add fluent https://fluent.github.io/helm-charts helm repo update --fail-on-repo-update-fail helm upgrade --install --debug --create-namespace --namespace "$TEST_NAMESPACE" fluent-bit fluent/fluent-bit \ --values ${BATS_TEST_DIRNAME}/resources/fluentbit-basic.yaml \ - --set image.repository=${FLUENTBIT_IMAGE_REPOSITORY},image.tag=${FLUENTBIT_IMAGE_TAG} \ + --set image.repository=${FLUENTBIT_IMAGE_REPOSITORY},image.tag=${FLUENTBIT_IMAGE_TAG},env[0].name=TEST_NAMESPACE,env[0].value=${TEST_NAMESPACE} \ --values "$HELM_VALUES_EXTRA_FILE" \ --timeout "${HELM_FB_TIMEOUT:-5m0s}" \ --wait @@ -66,34 +54,31 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" "to find 1 pods named 'fluent-bit' " \ "with 'status' being 'running'" + # The hello-world-1 container MUST be on the same node as the fluentbit worker, so we use a nodeSelector to specify the same node name + run kubectl get pods -l "app.kubernetes.io/name=fluent-bit" -o jsonpath='{.items[0].spec.nodeName}' + node_name=$output + + run kubectl run -n $TEST_NAMESPACE hello-world-1 --image=docker.io/library/alpine:latest -l "this_is_a_test_label=true" \ + --overrides="{\"apiVersion\":\"v1\",\"spec\":{\"nodeSelector\":{\"kubernetes.io/hostname\":\"$node_name\"}}}" \ + --command -- sh -c 'while true; do echo "hello world"; sleep 2; done' + + sleep 5 + # Wait for initial data to be reported attempt=0 while true; do - run kubectl exec -q -n "$TEST_NAMESPACE" elasticsearch-master-0 -- curl --insecure -s -w "%{http_code}" http://localhost:9200/fluentbit/_search/ -o /dev/null - if [[ "$output" != "200" ]]; then - if [ "$attempt" -lt 25 ]; then - attempt=$(( attempt + 1 )) - sleep 5 - else - fail "did not find any index results even after $attempt attempts" - fi - else - break - fi - done + run kubectl logs -l "app.kubernetes.io/name=fluent-bit" -n "$TEST_NAMESPACE" - # Search for what we want - attempt=0 - while true; do - search="http://localhost:9200/fluentbit/_search/?q=kubernetes.namespace_name:*&size=1&filter_path=hits.hits._source.kubernetes.labels" - run kubectl exec -q -n "$TEST_NAMESPACE" elasticsearch-master-0 -- curl --insecure -s "$search" - - if [[ "$output" != *'{"kubernetes":{"labels":'* ]]; then - if [ "$attempt" -lt 25 ]; then + match1='kubernetes":{"pod_name":"hello-world-1","namespace_name":' + match1=${match1}{$TEST_NAMESPACE} + match2='"labels":{"this_is_a_test_label":"true"}' + + if [[ "$output" != *${match1}*${match2}* && "$output" != *${match2}* ]]; then + if [ "$attempt" -lt 15 ]; then attempt=$(( attempt + 1 )) sleep 5 else - fail "did not find any kubernetes pod labels in output after $attempt attempts" + fail "did not find any kubernetes enhanced records even after $attempt attempts" fi else break diff --git a/tests/kubernetes-plugins/resources/elasticsearch-basic.yaml b/tests/kubernetes-plugins/resources/elasticsearch-basic.yaml deleted file mode 100644 index 3de9b97..0000000 --- a/tests/kubernetes-plugins/resources/elasticsearch-basic.yaml +++ /dev/null @@ -1,13 +0,0 @@ -protocol: http -httpPort: 9200 -transportPort: 9300 -replicas: 1 -minimumMasterNodes: 1 -createCert: false -extraEnvs: - - name: xpack.security.enabled - value: "false" - - name: xpack.security.http.ssl.enabled - value: "false" - - name: xpack.security.transport.ssl.enabled - value: "false" diff --git a/tests/kubernetes-plugins/resources/fluentbit-basic.yaml b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml index 7800633..1022b06 100644 --- a/tests/kubernetes-plugins/resources/fluentbit-basic.yaml +++ b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml @@ -5,21 +5,17 @@ rbac: extraVolumeMounts: - mountPath: /var/log name: varlog -- mountPath: /output - name: output extraVolumes: - name: varlog hostPath: path: /var/log -- name: output - emptyDir: - sizeLimit: 5Mi + config: service: | [SERVICE] Flush 5 Daemon Off - Log_Level debug + Log_Level error HTTP_Server On HTTP_Listen 0.0.0.0 HTTP_Port 2020 @@ -28,12 +24,13 @@ config: [INPUT] name tail read_from_head true - path /var/log/containers/*.log + path /var/log/containers/*${TEST_NAMESPACE}*.log multiline.parser docker, cri Tag kube.* buffer_chunk_size 2M buffer_max_size 2M Exclude_Path /var/log/containers/fluent-bit* + Refresh_Interval 5 filters: | [FILTER] @@ -49,8 +46,13 @@ config: outputs: | [OUTPUT] - Name es + Name stdout Match * + Format json_lines + + [OUTPUT] + Name es + Match *NONE Host elasticsearch-master Port 9200 Index fluentbit \ No newline at end of file From 5d21715cb940576304e9e4ae95ef2c7d031b1613 Mon Sep 17 00:00:00 2001 From: ryanohnemus Date: Fri, 12 Jan 2024 08:24:48 -0600 Subject: [PATCH 3/4] update bats-detik lib and address review comments Signed-off-by: ryanohnemus --- tests/kubernetes-plugins/basic.bats | 48 +++++++++---------- .../resources/fluentbit-basic.yaml | 9 +--- tools/install-bats.sh | 2 +- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/tests/kubernetes-plugins/basic.bats b/tests/kubernetes-plugins/basic.bats index 3024015..63270b3 100644 --- a/tests/kubernetes-plugins/basic.bats +++ b/tests/kubernetes-plugins/basic.bats @@ -11,7 +11,7 @@ load "$BATS_SUPPORT_ROOT/load.bash" load "$BATS_ASSERT_ROOT/load.bash" load "$BATS_FILE_ROOT/load.bash" -setup_file() { +setup() { echo "recreating namespace $TEST_NAMESPACE" run kubectl delete namespace "$TEST_NAMESPACE" run kubectl create namespace "$TEST_NAMESPACE" @@ -25,7 +25,7 @@ setup_file() { fi } -teardown_file() { +teardown() { if [[ "${SKIP_TEARDOWN:-no}" != "yes" ]]; then run kubectl delete namespace "$TEST_NAMESPACE" rm -f ${HELM_VALUES_EXTRA_FILE} @@ -56,32 +56,28 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" # The hello-world-1 container MUST be on the same node as the fluentbit worker, so we use a nodeSelector to specify the same node name run kubectl get pods -l "app.kubernetes.io/name=fluent-bit" -o jsonpath='{.items[0].spec.nodeName}' + assert_success node_name=$output run kubectl run -n $TEST_NAMESPACE hello-world-1 --image=docker.io/library/alpine:latest -l "this_is_a_test_label=true" \ --overrides="{\"apiVersion\":\"v1\",\"spec\":{\"nodeSelector\":{\"kubernetes.io/hostname\":\"$node_name\"}}}" \ - --command -- sh -c 'while true; do echo "hello world"; sleep 2; done' - - sleep 5 - - # Wait for initial data to be reported - attempt=0 - while true; do - run kubectl logs -l "app.kubernetes.io/name=fluent-bit" -n "$TEST_NAMESPACE" - - match1='kubernetes":{"pod_name":"hello-world-1","namespace_name":' - match1=${match1}{$TEST_NAMESPACE} - match2='"labels":{"this_is_a_test_label":"true"}' - - if [[ "$output" != *${match1}*${match2}* && "$output" != *${match2}* ]]; then - if [ "$attempt" -lt 15 ]; then - attempt=$(( attempt + 1 )) - sleep 5 - else - fail "did not find any kubernetes enhanced records even after $attempt attempts" - fi - else - break - fi - done + --command -- sh -c 'while true; do echo "hello world"; sleep 1; done' + + try "at most 15 times every 5s " \ + "to find 1 pods named 'hello-world-1' " \ + "with 'status' being 'Running'" + + # We are sleeping here specifically for the Fluent-Bit's tail input's + # configured Refresh_Interval to have enough time to detect the new pod's log file + # and to have processed part of it + sleep 3 + + run kubectl logs -l "app.kubernetes.io/name=fluent-bit" -n "$TEST_NAMESPACE" + match1='kubernetes":{"pod_name":"hello-world-1","namespace_name":' + match1=${match1}\"${TEST_NAMESPACE}\" + match2='"labels":{"this_is_a_test_label":"true"}' + + assert_output --partial $match1 + assert_output --partial $match2 + } \ No newline at end of file diff --git a/tests/kubernetes-plugins/resources/fluentbit-basic.yaml b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml index 1022b06..2551318 100644 --- a/tests/kubernetes-plugins/resources/fluentbit-basic.yaml +++ b/tests/kubernetes-plugins/resources/fluentbit-basic.yaml @@ -30,7 +30,7 @@ config: buffer_chunk_size 2M buffer_max_size 2M Exclude_Path /var/log/containers/fluent-bit* - Refresh_Interval 5 + Refresh_Interval 1 filters: | [FILTER] @@ -49,10 +49,3 @@ config: Name stdout Match * Format json_lines - - [OUTPUT] - Name es - Match *NONE - Host elasticsearch-master - Port 9200 - Index fluentbit \ No newline at end of file diff --git a/tools/install-bats.sh b/tools/install-bats.sh index 6b5586e..26dcfae 100755 --- a/tools/install-bats.sh +++ b/tools/install-bats.sh @@ -31,7 +31,7 @@ mkdir -p "${BATS_ROOT}/lib" BATS_ASSERT_VERSION=${BATS_ASSERT_VERSION:-2.0.0} BATS_SUPPORT_VERSION=${BATS_SUPPORT_VERSION:-0.3.0} BATS_FILE_VERSION=${BATS_FILE_VERSION:-0.3.0} -BATS_DETIK_VERSION=${BATS_DETIK_VERSION:-1.0.0} +BATS_DETIK_VERSION=${BATS_DETIK_VERSION:-1.2.1} DOWNLOAD_TEMP_DIR=$(mktemp -d) From 09b89e32ebada9d18ac1d90de427c1fb53049dd4 Mon Sep 17 00:00:00 2001 From: ryanohnemus Date: Fri, 12 Jan 2024 09:35:48 -0600 Subject: [PATCH 4/4] address pr comments Signed-off-by: ryanohnemus --- tests/kubernetes-plugins/basic.bats | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/kubernetes-plugins/basic.bats b/tests/kubernetes-plugins/basic.bats index 63270b3..f60245c 100644 --- a/tests/kubernetes-plugins/basic.bats +++ b/tests/kubernetes-plugins/basic.bats @@ -57,9 +57,10 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" # The hello-world-1 container MUST be on the same node as the fluentbit worker, so we use a nodeSelector to specify the same node name run kubectl get pods -l "app.kubernetes.io/name=fluent-bit" -o jsonpath='{.items[0].spec.nodeName}' assert_success + refute_output "" node_name=$output - run kubectl run -n $TEST_NAMESPACE hello-world-1 --image=docker.io/library/alpine:latest -l "this_is_a_test_label=true" \ + kubectl run -n $TEST_NAMESPACE hello-world-1 --image=docker.io/library/alpine:latest -l "this_is_a_test_label=true" \ --overrides="{\"apiVersion\":\"v1\",\"spec\":{\"nodeSelector\":{\"kubernetes.io/hostname\":\"$node_name\"}}}" \ --command -- sh -c 'while true; do echo "hello world"; sleep 1; done' @@ -69,10 +70,14 @@ DETIK_CLIENT_NAMESPACE="${TEST_NAMESPACE}" # We are sleeping here specifically for the Fluent-Bit's tail input's # configured Refresh_Interval to have enough time to detect the new pod's log file - # and to have processed part of it - sleep 3 + # and to have processed part of it. + # A future improvement instead of sleep could use fluentbit's metrics endpoints + # to know the tail lugin has processed records + sleep 10 run kubectl logs -l "app.kubernetes.io/name=fluent-bit" -n "$TEST_NAMESPACE" + assert_success + refute_output "" match1='kubernetes":{"pod_name":"hello-world-1","namespace_name":' match1=${match1}\"${TEST_NAMESPACE}\" match2='"labels":{"this_is_a_test_label":"true"}'