AppMesh on k8s load test

walkthroughs/howto-k8s-appmesh-load-test/.gitignore

@@ -0,0 +1,3 @@
+./logs
+.idea/
+scripts/data

walkthroughs/howto-k8s-appmesh-load-test/cluster.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+
+metadata:
+  name: basic-cluster
+  region: us-west-2
+
+nodeGroups:
+  - name: ng-1
+    instanceType: m5.4xlarge
+    desiredCapacity: 4
+    volumeSize: 80
+    ssh:
+      allow: true # will use ~/.ssh/id_rsa.pub as the default ssh key

walkthroughs/howto-k8s-appmesh-load-test/config.json

@@ -0,0 +1,22 @@
+{
+  "IsTLSEnabled": true,
+  "IsmTLSEnabled": false,
+  "ReplicasPerVirtualNode": 1,
+  "ConnectivityCheckPerURL": 400,
+  "backends_map": {
+                    "0": ["1", "2"],
+                    "1": ["3"],
+                    "2": ["4"]
+                },
+  "load_tests": [
+    {"test_name": "experiment_x", "url": "http://service-0.tls-e2e.svc.cluster.local:9080/", "qps": "5000", "t": "10s", "c":"400"}
+  ],
+  "metrics": {
+    "envoy_ingress_rate_by_replica_set": "sum(rate(envoy_cluster_upstream_rq{job=\"appmesh-envoy\",kubernetes_pod_name=~\"node-.*\",envoy_cluster_name=~\"cds_ingress.*\"}[10s])) by (kubernetes_pod_name)",
+    "envoy_2xx_requests_rate_by_replica_set": "sum(rate(envoy_cluster_upstream_rq{job=\"appmesh-envoy\",kubernetes_pod_name=~\"node-.*\",envoy_response_code=~\"2.*\",envoy_cluster_name=~\"cds_ingress.*\"}[10s])) by (kubernetes_pod_name)",
+    "envoy_4xx_requests_rate_by_replica_set": "sum(rate(envoy_cluster_upstream_rq{job=\"appmesh-envoy\",kubernetes_pod_name=~\"node-.*\",envoy_response_code=~\"4.*\",envoy_cluster_name=~\"cds_ingress.*\"}[10s])) by (kubernetes_pod_name)",
+    "envoy_5xx_requests_rate_by_replica_set": "sum(rate(envoy_cluster_upstream_rq{job=\"appmesh-envoy\",kubernetes_pod_name=~\"node-.*\",envoy_response_code=~\"5.*\",envoy_cluster_name=~\"cds_ingress.*\"}[10s])) by (kubernetes_pod_name)",
+    "envoy_memory_MB_by_replica_set": "sum(label_replace(container_memory_working_set_bytes{container=\"envoy\",pod=~\"node-.*\"}, \"replica_set\", \"$1\", \"pod\", \"(node-.*-.*)-.*\")) by (replica_set) / (1024*1024)",
+    "envoy_cpu_usage_seconds_by_replica_set": "sum(label_replace(container_cpu_usage_seconds_total{container=\"envoy\",pod=~\"node-.*\"}, \"replica_set\", \"$1\", \"pod\", \"(node-.*-.*)-.*\")) by (replica_set)"
+  }
+}

walkthroughs/howto-k8s-appmesh-load-test/configmap.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: scripts-configmap
+data:
+  request_handler_driver.sh: "<path-to>/scripts/request_handler_driver.sh"

walkthroughs/howto-k8s-appmesh-load-test/fortio.yaml

@@ -0,0 +1,57 @@
+---
+apiVersion: appmesh.k8s.aws/v1beta2
+kind: VirtualNode
+metadata:
+  name: fortio
+  namespace: tls-e2e
+spec:
+  podSelector:
+    matchLabels:
+      app: fortio
+  listeners:
+    - portMapping:
+        port: 8080
+        protocol: http
+  backends:
+    - virtualService:
+        virtualServiceRef:
+          name: service-0       # Replace with one of the fishap VS
+          namespace: tls-e2e
+  serviceDiscovery:
+    dns:
+      hostname: fortio.tls-e2e.svc.cluster.local
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: fortio
+  namespace: tls-e2e
+spec:
+  ports:
+    - port: 8080
+      name: http
+  selector:
+    app: fortio
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fortio
+  namespace: tls-e2e
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: fortio
+  template:
+    metadata:
+      labels:
+        app: fortio
+    spec:
+      containers:
+        - name: app
+          image: fortio/fortio
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 8080
+          args: ["server"]

walkthroughs/howto-k8s-appmesh-load-test/requirements.txt

@@ -0,0 +1,7 @@
+altair==4.2.0
+boto3==1.26.14
+botocore==1.29.14
+matplotlib==3.5.3
+numpy==1.21.6
+pandas==1.3.5
+requests==2.28.1

walkthroughs/howto-k8s-appmesh-load-test/scripts/analyze_load_test_data.py

@@ -0,0 +1,106 @@
+import csv
+import json
+import os
+import subprocess
+from collections import OrderedDict
+from pathlib import Path
+from pprint import pprint
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from constants import S3_BUCKET
+
+DIR_PATH = os.path.dirname(os.path.realpath(__file__))
+DATA_PATH = os.path.join(DIR_PATH, 'data')
+
+NODE_NAME = "0"
+
+
+def get_s3_data():
+    res = subprocess.run(["aws sts get-caller-identity"], shell=True, stdout=subprocess.PIPE,
+                         universal_newlines=True)
+    out = res.stdout
+    print("Caller identity: {}".format(out))
+
+    command = "aws s3 sync s3://{} {}".format(S3_BUCKET, DATA_PATH)
+    print("Running the following command to download S3 load test results:  \n{}".format(command))
+    res = subprocess.run([command], shell=True, stdout=subprocess.PIPE, universal_newlines=True)
+    out = res.stdout
+    print(out)
+
+
+def plot_graph(actual_QPS_list, node_0_mem_list):
+    x_y_tuple = [(x, y) for x, y in sorted(zip(actual_QPS_list, node_0_mem_list))]
+    X, Y = zip(*x_y_tuple)
+    xpoints = np.array(X)
+    ypoints = np.array(Y)
+
+    plt.figure(figsize=(10, 5))
+    plt.bar(xpoints, ypoints, width=20)
+    plt.ylabel('Node-{} (MiB)'.format(NODE_NAME))
+    plt.xlabel('Actual QPS')
+    print("Plotting graph...")
+    plt.show()
+
+
+def read_load_test_data():
+    all_files_list = [x for x in os.listdir(DATA_PATH) if os.path.isdir(os.path.join(DATA_PATH, x))]
+    qps_mem_files_list = []
+    for exp_f in all_files_list:
+        result = [os.path.join(dp, f) for dp, dn, filenames in os.walk(os.path.join(DATA_PATH, exp_f)) for f in
+                  filenames
+                  if "fortio.json" in f or "envoy_memory_MB_by_replica_set.csv" in f]
+        qps_mem_files_list.append(result)
+
+    actual_qps_list = []
+    node_0_mem_list = []
+    experiment_results = {}
+    for qps_or_mem_f in qps_mem_files_list:
+        attrb = {}
+        for f in qps_or_mem_f:
+            if "fortio.json" in f:
+                with open(f) as json_f:
+                    j = json.load(json_f)
+                    actual_qps = j["ActualQPS"]
+                    actual_qps_list.append(actual_qps)
+                    attrb["ActualQPS"] = actual_qps
+            else:
+                with open(f) as csv_f:
+                    c = csv.reader(csv_f, delimiter=',', skipinitialspace=True)
+                    node_0_mem = []
+                    node_found = False
+                    for line in c:
+                        if "node-" + NODE_NAME in line[0]:
+                            node_0_mem.append(float(line[2]))
+                            node_found = True
+                    if not node_found:
+                        raise Exception("Node not found: {} in experiment file: {}".format(NODE_NAME, f))
+                    max_mem = max(node_0_mem)
+                    node_0_mem_list.append(max_mem)
+                    attrb["max_mem"] = max_mem
+            key = Path(f)
+            experiment_results[os.path.join(key.parts[-3], key.parts[-2])] = attrb
+
+    # for research purpose
+    sorted_experiment_results = OrderedDict()
+    for k, v in sorted(experiment_results.items(), key=lambda item: item[1]['max_mem']):
+        sorted_experiment_results[k] = v
+    print("Experiment results sorted:")
+    pprint(sorted_experiment_results)
+
+    return actual_qps_list, node_0_mem_list
+
+
+def plot_qps_vs_container_mem():
+    actual_qps_list, node_0_mem_list = read_load_test_data()
+
+    plot_graph(actual_qps_list, node_0_mem_list)
+
+
+if __name__ == '__main__':
+    node_name = input('Enter the node name (or press enter for default node "0"): ')
+    if node_name != "":
+        NODE_NAME = node_name
+    get_s3_data()
+    plot_qps_vs_container_mem()

walkthroughs/howto-k8s-appmesh-load-test/scripts/constants.py

@@ -0,0 +1,10 @@
+URL_DEFAULT = "http://service-0.tls-e2e.svc.cluster.local:9080/path-0"
+QPS_DEFAULT = "100"
+DURATION_DEFAULT = "30s"
+CONNECTIONS_DEFAULT = "1"
+
+FORTIO_RUN_ENDPOINT = 'http://localhost:9091/fortio/rest/run'
+PROMETHEUS_QUERY_ENDPOINT = 'http://localhost:9090/api/v1/query_range'
+
+# give your s3 bucket a unique name
+S3_BUCKET = "<username>-appmeshloadtester"

walkthroughs/howto-k8s-appmesh-load-test/scripts/driver.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+err() {
+    msg="Error: $1"
+    echo "${msg}"
+    code=${2:-"1"}
+    exit ${code}
+}
+
+exec_command() {
+  eval "$1"
+  if [ $? -eq 0 ]; then
+      echo "'$1' command Executed Successfully"
+  else
+      err "'$1' command Failed"
+  fi
+}
+
+check_version() {
+  eval "$1"
+}
+
+# sanity check
+if [ -z "${CONTROLLER_PATH}" ]; then
+    err "CONTROLLER_PATH is not set"
+fi
+
+if [ -z "${KUBECONFIG}" ]; then
+    err "KUBECONFIG is not set"
+fi
+
+if [ -z "${CLUSTER_NAME}" ]; then
+    err "CLUSTER_NAME is not set"
+fi
+
+if [ -z "${AWS_REGION}" ]; then
+    err "AWS_REGION is not set"
+fi
+
+if [ -z "${VPC_ID}" ]; then
+    err "VPC_ID is not set"
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+APPMESH_LOADTESTER_PATH="$(dirname "$DIR")"
+echo "APPMESH_LOADTESTER_PATH -: $APPMESH_LOADTESTER_PATH"
+
+# Prometheus port forward
+echo "Port-forwarding Prometheus"
+kubectl --namespace appmesh-system port-forward service/appmesh-prometheus 9090 &
+pid=$!
+
+# call ginkgo
+echo "Starting Ginkgo test. This may take a while! So hang tight and do not close this window"
+cd $CONTROLLER_PATH && ginkgo -v -r --focus "DNS" "$CONTROLLER_PATH"/test/e2e/fishapp/load -- --cluster-kubeconfig=$KUBECONFIG \
+--cluster-name=$CLUSTER_NAME --aws-region=$AWS_REGION --aws-vpc-id=$VPC_ID \
+--base-path=$APPMESH_LOADTESTER_PATH
+
+# kill prometheus port forward
+echo "Killing Prometheus port-forward"
+kill -9 $pid
+[ $status -eq 0 ] && echo "Killed Prometheus port-forward" || echo "Error when killing Prometheus port forward"
+
+cd "$DIR" || exit