test: add metrics logging in e2e test

add metrics logging
kubernetes-csi · Aug 30, 2020 · 4754601 · 4754601
1 parent 92489ff
commit 4754601
Show file tree

Hide file tree

Showing 15 changed files with 67 additions and 7 deletions.
diff --git a/Makefile b/Makefile
@@ -158,3 +158,7 @@ ifdef TEST_WINDOWS
 else
 	kubectl create -f deploy/example/smb-provisioner/smb-server.yaml
 endif
+
+.PHONY: create-metrics-svc
+create-metrics-svc:
+	kubectl create -f deploy/example/metrics/csi-smb-controller-svc.yaml
diff --git a/charts/README.md b/charts/README.md
@@ -59,7 +59,9 @@ The following table lists the configurable parameters of the latest SMB CSI Driv
 | `imagePullSecrets`                                | Specify docker-registry secret names as an array           | [] (does not add image pull secrets to deployed pods)             |
 | `serviceAccount.create`                           | whether create service account of csi-smb-controller       | true                                                              |
 | `rbac.create`                                     | whether create rbac of csi-smb-controller                  | true                                                              |
-| `controller.replicas`                             | the replicas of csi-smb-controller                         | 2                                                                 |
+| `controller.replicas`                             | the replicas of csi-smb-controller                                  | 2                                                   |
+| `controller.metricsPort`                          | metrics port of csi-smb-controller                   |29644                                               |
+| `node.metricsPort`                                | metrics port of csi-smb-node                         |29645
 | `linux.enabled`                                   | whether enable linux feature                               | true                                                              |
 | `windows.enabled`                                 | whether enable windows feature                             | false                                                             |
 | `windows.image.livenessProbe.repository`          | windows liveness-probe docker image                        | mcr.microsoft.com/oss/kubernetes-csi/livenessprobe                |

diff --git a/charts/latest/csi-driver-smb/templates/csi-smb-controller.yaml b/charts/latest/csi-driver-smb/templates/csi-smb-controller.yaml
@@ -71,11 +71,12 @@ spec:
           args:
             - "--v=5"
             - "--endpoint=$(CSI_ENDPOINT)"
+            - "--metrics-address=0.0.0.0:{{ .Values.controller.metricsPort }}"
           ports:
             - containerPort: 29642
               name: healthz
               protocol: TCP
-            - containerPort: 29644
+            - containerPort: {{ .Values.controller.metricsPort }}
               name: metrics
               protocol: TCP
           livenessProbe:

diff --git a/charts/latest/csi-driver-smb/templates/csi-smb-node-windows.yaml b/charts/latest/csi-driver-smb/templates/csi-smb-node-windows.yaml
@@ -75,11 +75,12 @@ spec:
             - --v=5
             - --endpoint=$(CSI_ENDPOINT)
             - --nodeid=$(KUBE_NODE_NAME)
+            - "--metrics-address=0.0.0.0:{{ .Values.node.metricsPort }}"
           ports:
             - containerPort: 29643
               name: healthz
               protocol: TCP
-            - containerPort: 29645
+            - containerPort: {{ .Values.node.metricsPort }}
               name: metrics
               protocol: TCP
           livenessProbe:

diff --git a/charts/latest/csi-driver-smb/templates/csi-smb-node.yaml b/charts/latest/csi-driver-smb/templates/csi-smb-node.yaml
@@ -74,11 +74,12 @@ spec:
             - "--v=5"
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodeid=$(KUBE_NODE_NAME)"
+            - "--metrics-address=0.0.0.0:{{ .Values.node.metricsPort }}"
           ports:
             - containerPort: 29643
               name: healthz
               protocol: TCP
-            - containerPort: 29645
+            - containerPort: {{ .Values.node.metricsPort }}
               name: metrics
               protocol: TCP
           livenessProbe:

diff --git a/charts/latest/csi-driver-smb/values.yaml b/charts/latest/csi-driver-smb/values.yaml
@@ -24,6 +24,10 @@ rbac:
 
 controller:
   replicas: 2
+  metricsPort: 29644
+
+node:
+  metricsPort: 29645
 
 linux:
   enabled: true

diff --git a/deploy/csi-smb-controller.yaml b/deploy/csi-smb-controller.yaml
@@ -67,6 +67,7 @@ spec:
           args:
             - "--v=5"
             - "--endpoint=$(CSI_ENDPOINT)"
+            - "--metrics-address=0.0.0.0:29644"
           ports:
             - containerPort: 29642
               name: healthz

diff --git a/deploy/csi-smb-node-windows.yaml b/deploy/csi-smb-node-windows.yaml
@@ -71,6 +71,7 @@ spec:
             - --v=5
             - --endpoint=$(CSI_ENDPOINT)
             - --nodeid=$(KUBE_NODE_NAME)
+            - "--metrics-address=0.0.0.0:29645"
           ports:
             - containerPort: 29643
               name: healthz

diff --git a/deploy/csi-smb-node.yaml b/deploy/csi-smb-node.yaml
@@ -70,6 +70,7 @@ spec:
             - "--v=5"
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--nodeid=$(KUBE_NODE_NAME)"
+            - "--metrics-address=0.0.0.0:29645"
           ports:
             - containerPort: 29643
               name: healthz

diff --git a/deploy/example/metrics/README.md b/deploy/example/metrics/README.md
@@ -0,0 +1,19 @@
+# Get Prometheus metrics from CSI driver
+
+1. Create `csi-smb-controller` service with targetPort `29644`
+```console
+kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/csi-driver-smb/master/deploy/example/metrics/csi-smb-controller-svc.yaml
+```
+
+2. Get `EXTERNAL-IP` of service `csi-smb-controller`
+```console
+$ kubectl get svc csi-smb-controller -n kube-system
+NAME                 TYPE           CLUSTER-IP     EXTERNAL-IP   PORT(S)           AGE
+csi-smb-controller   LoadBalancer   10.0.217.224   20.39.0.91    29644:32128/TCP   45m
+```
+
+3. Run following command to get metrics
+```console
+ip=`kubectl get svc csi-smb-controller -n kube-system | grep smb | awk '{print $4}'`
+curl http://$ip:29644/metrics
+```
diff --git a/deploy/example/metrics/csi-smb-controller-svc.yaml b/deploy/example/metrics/csi-smb-controller-svc.yaml
@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: csi-smb-controller
+  namespace: kube-system
+spec:
+  selector:
+    app: csi-smb-controller
+  ports:
+    - port: 29644
+      targetPort: 29644
+  type: LoadBalancer
diff --git a/hack/verify-yamllint.sh b/hack/verify-yamllint.sh
@@ -20,7 +20,7 @@ fi
 
 LOG=/tmp/yamllint.log
 
-for path in "deploy/*.yaml" "deploy/example/*.yaml" "deploy/example/windows/*.yaml" "deploy/example/smb-provisioner/*.yaml"
+for path in "deploy/*.yaml" "deploy/example/*.yaml" "deploy/example/windows/*.yaml" "deploy/example/smb-provisioner/*.yaml" "deploy/example/metrics/*.yaml"
 do
     echo "checking yamllint under path: $path ..."
     yamllint -f parsable $path | grep -v "line too long" > $LOG

diff --git a/pkg/smbplugin/main.go b/pkg/smbplugin/main.go
@@ -39,7 +39,7 @@ var (
 	endpoint       = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
 	nodeID         = flag.String("nodeid", "", "node id")
 	version        = flag.Bool("version", false, "Print the version and exit.")
-	metricsAddress = flag.String("metrics-address", "0.0.0.0:39615", "export the metrics")
+	metricsAddress = flag.String("metrics-address", "0.0.0.0:29644", "export the metrics")
 	kubeconfig     = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.")
 )
 

diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go
@@ -94,7 +94,14 @@ var _ = ginkgo.BeforeSuite(func() {
 			endLog:   "SMB CSI Driver installed",
 		}
 
-		execTestCmd([]testCmd{installSMBProvisioner, e2eBootstrap})
+		createMetricsSVC := testCmd{
+			command:  "make",
+			args:     []string{"create-metrics-svc"},
+			startLog: "create metrics service ...",
+			endLog:   "metrics service created",
+		}
+
+		execTestCmd([]testCmd{installSMBProvisioner, e2eBootstrap, createMetricsSVC})
 
 		nodeid := os.Getenv("nodeid")
 		kubeconfig := os.Getenv(kubeconfigEnvVar)

diff --git a/test/utils/smb_log.sh b/test/utils/smb_log.sh
@@ -47,3 +47,8 @@ LABEL='app=csi-smb-node-win'
 kubectl get pods -n${NS} -l${LABEL} \
     | awk 'NR>1 {print $1}' \
     | xargs -I {} kubectl logs {} --prefix -c${CONTAINER} -n${NS}
+
+echo "print out metrics ..."
+echo "======================================================================================"
+ip=`kubectl get svc csi-smb-controller -n kube-system | grep smb | awk '{print $4}'`
+curl http://$ip:29644/metrics