maint: adding proper smoke test(s) (#310)

## Which problem is this PR solving?

- Closes #78 

## Description of the changes

Set up a smoke test, details included in the new make targets below:
- `smokey_cluster_create`: Create kind cluster
- `smokey_collector_install`: Install collector via helm with new
collector values, including a filter processor that only keeps
echoserver spans from pods with a response code of 405
- `smokey_agent_install`: Install agent via helm with new agent values,
including usage of a local image (installed if not available in the
shell) and export to the collector
- `smokey_echo_job`: Install echoserver and run a job that sends a POST
request expecting a 405 response
- `smokey_copy_output`: After a short wait, copy output from the newly
created file saved with telemetry from collector
- `smokey_verify_output`: Make assertions via the new `verify.bats` bats
test (similar to previous tests on other distros and in go
auto-instrumentation) that the output contains the expected span and
attributes
- `smoke`: Do all of the things above
- `unsmoke`: Tear it all down when done

Also updated PR, main, and release workflows to run the new smoke test.

## How to verify that this has the expected result

- Run `make smoke` and get a bunch of passing tests - also check out the
(git ignored) json file that contains the remaining span that's being
tested. `make unsmoke` to spin it down.

---------

Co-authored-by: Mike Goldsmth <goldsmith.mike@gmail.com>
Co-authored-by: Jamie Danielson <jamiedanielson@honeycomb.io>
Co-authored-by: JamieDanielson <jamieedanielson@gmail.com>

.github/workflows/main.yaml

@@ -16,6 +16,15 @@ jobs:
       - name: Run tests
         run: make docker-test
 
+      - name: Setup BATS
+        uses: mig4/setup-bats@v1
+
+      - name: Run smoke tests
+        run: make smoke
+
+      - name: Spin down smokers
+        run: make unsmoke
+
       - name: Log in to the Container registry
         uses: docker/login-action@v2.2.0
         with:

.github/workflows/pr.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
 
 jobs:
-  docker-build:
+  build-and-test:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout Repo
@@ -13,18 +13,11 @@ jobs:
       - name: Run tests
         run: make docker-test
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2.2.0
+      - name: Setup BATS
+        uses: mig4/setup-bats@v1
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2.9.0
+      - name: Run smoke tests
+        run: make smoke
 
-      - name: Build docker image
-        uses: docker/build-push-action@v4.1.1
-        with:
-          context: .
-          push: false
-          platforms: linux/amd64
-          # platforms: linux/amd64,linux/arm64 - don't build arm64 on PRs for now
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+      - name: Spin down smokers
+        run: make unsmoke

.github/workflows/release.yaml

@@ -13,6 +13,18 @@ jobs:
     steps:
       - uses: actions/checkout@v3.5.3
 
+      - name: Run tests
+        run: make docker-test
+
+      - name: Setup BATS
+        uses: mig4/setup-bats@v1
+
+      - name: Run smoke tests
+        run: make smoke
+
+      - name: Spin down smokers
+        run: make unsmoke
+
       - name: Log in to the Container registry
         uses: docker/login-action@v2.2.0
         with:

.gitignore

@@ -15,4 +15,7 @@ terraform.tfstate*
 .env*
 
 # python cache (locust)
-__pycache__/
+__pycache__/
+
+# don't track smoke test traces
+smoke-tests/**/traces-orig.json

Makefile

@@ -1,5 +1,6 @@
 IMG_NAME ?= hny/network-agent
 IMG_TAG ?= local
+
 .PHONY: build
 #: compile the agent executable
 build:
@@ -22,6 +23,64 @@ docker-test:
 
 ### Testing targets
 
+.PHONY: smoke
+#: run smoke tests - for local tests comment out docker-build to save time, for CI uncomment docker-build
+smoke:  smokey_cluster_create smokey_collector_install smokey_agent_install smokey_echo_job smokey_copy_output smokey_verify_output
+
+smokey_cluster_create:
+	kind create cluster
+
+smokey_collector_install:
+  # install opentelemetry collector using helm chart and wait for it to be ready
+	helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+	helm install smokey-collector open-telemetry/opentelemetry-collector --values smoke-tests/collector-helm-values.yaml
+	kubectl rollout status statefulset.apps/smokey-collector-opentelemetry-collector --timeout=60s
+
+# If the image doesn't exist, return the name of the make target to build it.
+# Use this as a prerequisite to any target that needs the image to exist.
+maybe_docker_build = $(if $(shell docker images -q $(IMG_NAME):$(IMG_TAG)),,docker-build)
+
+smokey_agent_install: $(maybe_docker_build)
+  # install network agent using helm chart using local build and wait for it to be ready
+	kind load docker-image $(IMG_NAME):$(IMG_TAG)
+	helm repo add honeycomb https://honeycombio.github.io/helm-charts
+	OTEL_COLLECTOR_IP="$(call get_collector_ip)" \
+		envsubst < smoke-tests/agent-helm-values.yaml | helm install smokey-agent honeycomb/network-agent --values -
+	kubectl rollout status daemonset.apps/smokey-agent-network-agent --timeout=10s
+
+smokey_copy_output:
+  # copy output from collector file to local machine
+  # this is ignored in .gitignore
+  # TODO use more reliable way to wait for file to be ready
+  # for now just sleeping a bit
+	sleep 12
+	kubectl cp -c filecp default/smokey-collector-opentelemetry-collector-0:/tmp/trace.json ./smoke-tests/traces-orig.json
+
+smokey_verify_output:
+  # verify that the output from the collector matches the expected output
+	bats ./smoke-tests/verify.bats
+
+# A function to get the IP address of the collector service. Use = instead of := so that it is lazy-evaluated.
+get_collector_ip = \
+	$(shell kubectl get service smokey-collector-opentelemetry-collector --template '{{.spec.clusterIP}}')
+
+.PHONY: smokey_echo_job
+#: apply echo server and run smoke-job; not necessary for local setup - plenty of chatter already
+smokey_echo_job:
+	make apply-echoserver
+	kubectl create --filename smoke-tests/smoke-job.yaml
+	kubectl rollout status deployment.apps/echoserver --timeout=10s --namespace echoserver
+	kubectl wait --for=condition=complete job/smoke-job --timeout=60s --namespace echoserver
+
+.PHONY: unsmoke
+#: teardown smoke tests
+unsmoke:
+	kind delete cluster
+
+.PHONY: resmoke
+#: run smoke tests again
+resmoke: unsmoke smoke
+
 .PHONY: apply-agent
 #: deploy network agent daemonset to already-running cluster with env vars from .env file
 apply-agent:

smoke-tests/agent-helm-values.yaml

@@ -0,0 +1,20 @@
+honeycomb:
+  apiKey: "bogosity" # we'll be sending to a Collector in smoke tests
+  endpoint: "http://$OTEL_COLLECTOR_IP:4317"
+
+image:
+  repository: hny/network-agent
+  # use local image for testing instead of helm chart default
+  tag: local
+
+extraEnvVars:
+  - name: OTEL_RESOURCE_ATTRIBUTES
+    value: "environment=smokey"
+  - name: INCLUDE_REQUEST_URL
+    value: "true"
+  - name: HTTP_HEADERS
+    value: "User-Agent,Traceparent,X-Custom-Header"
+  - name: OTEL_EXPORTER_OTLP_INSECURE
+    value: "true"
+  - name: LOG_LEVEL
+    value: "DEBUG"

smoke-tests/collector-helm-values.yaml

@@ -0,0 +1,70 @@
+mode: "statefulset"
+
+config:
+  receivers:
+    jaeger: null
+    prometheus: null
+    zipkin: null
+    otlp:
+      protocols:
+        http:
+          endpoint: ${env:MY_POD_IP}:4318
+        grpc:
+          endpoint: ${env:MY_POD_IP}:4317
+
+  processors:
+    filter/echoonly:
+      spans:
+        include:
+          match_type: strict
+          attributes:
+            - key: destination.k8s.namespace.name
+              value: echoserver
+        exclude:
+          match_type: strict
+          attributes:
+            - key: destination.k8s.resource.type
+              value: service
+
+  exporters:
+    debug:
+      verbosity: detailed
+    file/trace:
+      path: /tmp/trace.json
+      rotation:
+
+  service:
+    pipelines:
+      metrics: null
+      logs: null
+      traces:
+        receivers:
+          - otlp
+        processors:
+          - filter/echoonly
+        exporters:
+          - file/trace
+          - debug
+
+
+image:
+  repository: otel/opentelemetry-collector-contrib
+  tag: "latest"
+
+command:
+  name: otelcol-contrib
+
+extraVolumes:
+- name: filevolume
+  emptyDir: {}
+extraVolumeMounts:
+- mountPath: /tmp
+  name: filevolume
+
+extraContainers:
+- name: filecp
+  image: busybox
+  command: ["sh", "-c", "sleep 36000"]
+  volumeMounts:
+  - name: filevolume
+    mountPath: /tmp

smoke-tests/smoke-job.yaml

@@ -0,0 +1,29 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: smoke-job
+  namespace: echoserver
+spec:
+  template:
+    metadata:
+      annotations:
+        workload: job
+      labels:
+        app: smoke
+    spec:
+      shareProcessNamespace: true
+      restartPolicy: Never
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+      containers:
+      - name: smoke-curl
+        image: docker.io/curlimages/curl
+        imagePullPolicy: IfNotPresent
+        args:
+        - --verbose
+        - --data
+        - 'message=heygirl'
+        - --header
+        - 'X-ECHO-CODE: 405'
+        - http://echoserver

smoke-tests/test_helpers/utilities.bash

@@ -0,0 +1,74 @@
+# UTILITY FUNCS
+
+# Span names for a given scope
+# Arguments: $1 - scope name
+span_names_for() {
+	spans_from_scope_named $1 | jq '.name'
+}
+
+# Attributes for a given scope
+# Arguments: $1 - scope name
+span_attributes_for() {
+	spans_from_scope_named $1 | \
+		jq ".attributes[]"
+}
+
+# All resource attributes
+resource_attributes_received() {
+	spans_received | jq ".resource.attributes[]?"
+}
+
+# Spans for a given scope
+# Arguments: $1 - scope name
+spans_from_scope_named() {
+	spans_received | jq ".scopeSpans[] | select(.scope.name == \"$1\").spans[]"
+}
+
+# All spans received
+spans_received() {
+	json_output | jq ".resourceSpans[]?"
+}
+
+# the output of the log file produced by the collector
+json_output() {
+	cat "${BATS_TEST_DIRNAME}/traces-orig.json"
+}
+
+# ASSERTION HELPERS
+
+# Fail and display details if the expected and actual values do not
+# equal. Details include both values.
+#
+# Inspired by bats-assert * bats-support, but dramatically simplified
+# Arguments:
+# $1 - actual result
+# $2 - expected result
+assert_equal() {
+	if [[ $1 != "$2" ]]; then
+		{
+			echo
+			echo "-- 💥 values are not equal 💥 --"
+			echo "expected : $2"
+			echo "actual   : $1"
+			echo "--"
+			echo
+		} >&2 # output error to STDERR
+		return 1
+	fi
+}
+
+# Fail and display details if the actual value is empty.
+# Arguments: $1 - actual result
+assert_not_empty() {
+	EMPTY=(\"\")
+	if [[ "$1" == "${EMPTY}" ]]; then
+		{
+			echo
+			echo "-- 💥 value is empty 💥 --"
+			echo "value : $1"
+			echo "--"
+			echo
+		} >&2 # output error to STDERR
+		return 1
+	fi
+}