e2e: add test for dynamic balloons configuration update

- Add new test that changes balloons and CPU configurations using a
  config map.
- Refactor verify-metrics functions to a common source file.

test/e2e/policies.test-suite/balloons/balloons-configmap.yaml.in

@@ -0,0 +1,79 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cri-resmgr-config.default
+  namespace: kube-system
+data:
+  policy: |+
+    Active: balloons
+    AvailableResources:
+      CPU: ${AVAILABLE_CPU:-cpuset:0-15}
+    ReservedResources:
+      CPU: ${RESERVED_CPU:-1}
+
+    $([ -z "$IDLECPUCLASS" ] || echo "
+    IdleCPUClass: ${IDLECPUCLASS}
+    ")
+
+    balloons:
+      PinCPU: ${PINCPU:-true}
+      PinMemory: ${PINMEMORY:-true}
+      BalloonTypes:
+
+        $([ -n "$BTYPE0_SKIP" ] || echo "
+        - Name: btype0
+          MinCPUs: ${BTYPE0_MINCPUS:-2}
+          MaxCPUs: ${BTYPE0_MAXCPUS:-2}
+          AllocationPriority: ${BTYPE0_ALLOCATIONPRIORITY:-0}
+          CPUClass: ${BTYPE0_CPUCLASS:-classA}
+          PreferNewBalloons: ${BTYPE0_PREFERNEWBALLOONS:-true}
+          PreferSpreadingPods: ${BTYPE0_PREFERSPREADINGPODS:-false}
+        ")
+
+        $([ -n "$BTYPE1_SKIP" ] || echo "
+        - Name: btype1
+          Namespaces:
+            - ${BTYPE1_NAMESPACE0:-btype1ns0}
+          MinCPUs: ${BTYPE1_MINCPUS:-1}
+          AllocatorPriority: ${BTYPE1_ALLOCATIONPRIORITY:-1}
+          CPUClass: ${BTYPE1_CPUCLASS:-classB}
+          PreferNewBalloons: ${BTYPE1_PREFERNEWBALLOONS:-false}
+          PreferSpreadingPods: ${BTYPE1_PREFERSPREADINGPODS:-true}
+        ")
+
+        $([ -n "$BTYPE2_SKIP" ] || echo "
+        - Name: btype2
+          Namespaces:
+            - ${BTYPE2_NAMESPACE0:-btype2ns0}
+            - ${BTYPE2_NAMESPACE1:-btype2ns1}
+          MinCPUs: ${BTYPE2_MINCPUS:-4}
+          MaxCPUs: ${BTYPE2_MAXCPUS:-8}
+          MinBalloons: ${BTYPE2_MINBALLOONS:-1}
+          AllocatorPriority: ${BTYPE2_ALLOCATORPRIORITY:-2}
+          CPUClass: ${BTYPE2_CPUCLASS:-classC}
+          PreferNewBalloons: ${BTYPE2_PREFERNEWBALLOONS:-false}
+          PreferSpreadingPods: ${BTYPE2_PREFERSPREADINGPODS:-false}
+        ")
+
+  instrumentation: |+
+    HTTPEndpoint: :8891
+    PrometheusExport: true
+
+  logger: |+
+    Debug: policy
+
+  cpu: |+
+    classes:
+      default:
+        minFreq: ${CPU_DEFAULT_MIN:-800}
+        maxFreq: ${CPU_DEFAULT_MAX:-2800}
+      classA:
+        minFreq: ${CPU_CLASSA_MIN:-900}
+        maxFreq: ${CPU_CLASSA_MAX:-2900}
+      classB:
+        minFreq: ${CPU_CLASSB_MIN:-1000}
+        maxFreq: ${CPU_CLASSB_MAX:-3000}
+      classC:
+        minFreq: ${CPU_CLASSC_MIN:-1100}
+        maxFreq: ${CPU_CLASSC_MAX:-3100}
+        energyPerformancePreference: ${CPU_CLASSC_EPP:-1}

test/e2e/policies.test-suite/balloons/n4c16/test02-prometheus-metrics/code.var.sh

@@ -1,22 +1,5 @@
 # This test verifies prometheus metrics from the balloons policy.
 
-metrics_url="http://localhost:8891/metrics"
-
-verify-metrics-has-line() {
-    local expected_line="$1"
-    vm-run-until --timeout 10 "echo 'waiting for metrics line: $expected_line' >&2; curl --silent $metrics_url | grep -E '$expected_line'" || {
-        command-error "expected line '$1' missing from the output"
-    }
-}
-
-verify-metrics-has-no-line() {
-    local unexpected_line="$1"
-    vm-command "echo 'checking absense of metrics line: $unexpected_line' >&2; curl --silent $metrics_url | grep -E '$unexpected_line'" && {
-        command-error "unexpected line '$1' found from the output"
-    }
-    return 0
-}
-
 # Launch cri-resmgr with wanted metrics update interval and a
 # configuration that opens the instrumentation http server.
 terminate cri-resmgr

test/e2e/policies.test-suite/balloons/n4c16/test06-update-configmap/code.var.sh

@@ -0,0 +1,67 @@
+# This test verifies that configuration updates via cri-resmgr-agent
+# are handled properly in the balloons policy.
+
+testns=e2e-balloons-test06
+
+cleanup() {
+    kubectl delete pods --all --now
+    ( kubectl delete namespace $testns )
+    ( kubectl delete btype1ns0 )
+    terminate cri-resmgr
+    terminate cri-resmgr-agent
+    vm-command "cri-resmgr -reset-policy; cri-resmgr -reset-config"
+}
+
+apply-configmap() {
+    vm-put-file $(instantiate balloons-configmap.yaml) balloons-configmap.yaml
+    vm-command "cat balloons-configmap.yaml"
+    kubectl apply -f balloons-configmap.yaml
+}
+
+cleanup
+cri_resmgr_extra_args="-metrics-interval 1s" cri_resmgr_config=fallback launch cri-resmgr
+launch cri-resmgr-agent
+
+kubectl create namespace $testns
+kubectl create namespace btype1ns0
+
+AVAILABLE_CPU="cpuset:0,4-15" BTYPE2_NAMESPACE0='"*"' apply-configmap
+sleep 3
+
+# pod0 in btype0, annotation
+CPUREQ=1 MEMREQ="100M" CPULIM=1 MEMLIM="100M"
+POD_ANNOTATION="balloon.balloons.cri-resource-manager.intel.com: btype0" create balloons-busybox
+# pod1 in btype1, namespace
+CPUREQ=1 MEMREQ="100M" CPULIM=1 MEMLIM="100M"
+namespace="btype1ns0" create balloons-busybox
+# pod2 in btype2, wildcard namespace
+CPUREQ=1 MEMREQ="100M" CPULIM=1 MEMLIM="100M"
+namespace="e2e-balloons-test06" create balloons-busybox
+vm-command "curl -s $verify_metrics_url"
+verify-metrics-has-line 'btype0\[0\].*containers="pod0:pod0c0"'
+verify-metrics-has-line 'btype1\[0\].*containers="pod1:pod1c0"'
+verify-metrics-has-line 'btype2\[0\].*containers="pod2:pod2c0"'
+
+# Remove first two balloon types, change btype2 to match all
+# namespaces.
+BTYPE0_SKIP=1 BTYPE1_SKIP=1 BTYPE2_NAMESPACE0='"*"' apply-configmap
+# Note:
+
+# pod0 was successfully assigned to and running in balloon of btype0.
+# Now btype0 was completely removed from the node.
+# Currently this behavior is undefined.
+# Possible behaviors: evict pod0, continue assign chain, refuse config...
+# For now, skip pod0c0 balloon validation:
+# verify-metrics-has-line '"btype2\[0\]".*pod0:pod0c0'
+verify-metrics-has-line '"btype2\[0\]".*pod1:pod1c0'
+verify-metrics-has-line '"btype2\[0\]".*pod2:pod2c0'
+
+# Bring back btype0 where pod0 belongs to by annotation.
+BTYPE1_SKIP=1 BTYPE2_NAMESPACE0='"*"' apply-configmap
+verify-metrics-has-line '"btype0\[0\]".*pod0:pod0c0'
+verify-metrics-has-line '"btype2\[0\]".*pod1:pod1c0'
+verify-metrics-has-line '"btype2\[0\]".*pod2:pod2c0'
+
+# Change only CPU classes, no reassigning.
+BTYPE0_CPUCLASS="classC" BTYPE1_SKIP=1 BTYPE2_NAMESPACE0='"*"'  apply-configmap
+# TODO: verify that no new workload assigments have been made.

test/e2e/policies.test-suite/balloons/verify.source.sh

@@ -0,0 +1,18 @@
+# Utilities to verify data from metrics
+
+verify_metrics_url="http://localhost:8891/metrics"
+
+verify-metrics-has-line() {
+    local expected_line="$1"
+    vm-run-until --timeout 10 "echo 'waiting for metrics line: $expected_line' >&2; curl --silent $verify_metrics_url | grep -E '$expected_line'" || {
+        command-error "expected line '$1' missing from the output"
+    }
+}
+
+verify-metrics-has-no-line() {
+    local unexpected_line="$1"
+    vm-command "echo 'checking absense of metrics line: $unexpected_line' >&2; curl --silent $verify_metrics_url | grep -E '$unexpected_line'" && {
+        command-error "unexpected line '$1' found from the output"
+    }
+    return 0
+}