feat: enable node metrics port 9999 through otel-collector

hashgraph · Aug 29, 2023 · 1b7cc5c · 1b7cc5c
1 parent f5aadc5
commit 1b7cc5c
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 70 deletions.
diff --git a/charts/hedera-network/config-files/otel-collector-config.yaml b/charts/hedera-network/config-files/otel-collector-config.yaml
@@ -2,21 +2,23 @@ receivers:
   otlp:
     protocols:
       grpc:
-  hostmetrics:
-    collection_interval: "15s"
-    scrapers:
-      cpu:
-      disk:
-      load:
-      filesystem:
-      memory:
-      network:
-      paging:
-      processes:
+      http:
+
+  # Collect node metrics
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: 'node-metrics-scraper' # network-node metrics
+          scrape_interval: 5s # TODO make scrape interval configurable
+          static_configs:
+            - targets: [ '0.0.0.0:9999' ] # TODO make the node metrics port configurable
+
+processors:
+  batch:
 
 exporters:
   prometheus:
-    endpoint: "0.0.0.0:8889"
+    endpoint: "0.0.0.0:8889" # note: network-node-svc exposes this port for prometheus metrics instead of node's port 9999
     const_labels:
       source: p-{{ default "otel-collector" .otelDefaults.nameOverride }} # PromQL: {source="p-otel-collector"}
   {{- if eq .otelDefaults.prometheusRemoteWrite.enable "true" }}
@@ -27,43 +29,32 @@ exporters:
     external_labels:
       source: prw-{{ default "otel-collector" .otelDefaults.nameOverride }} # PromQL: {source="prw-otel-collector"}
   {{- end }}
+
 #  logging:
 #    verbosity: detailed
 #    sampling_initial: 5
 
-
-#  jaeger:
-#    endpoint: jaeger-all-in-one:14250
-#    tls:
-#      insecure: true
-#
 #  otlp:
-#    endpoint: tempo:4317
+#    endpoint: ""
 #    tls:
 #      insecure: true
 
-processors:
-  batch:
 extensions:
   health_check:
 
 service:
-#  telemetry:
-#    logs:
-#      level: "debug"
-#      development: true
   extensions: [health_check]
   pipelines:
-    #traces:
-    #  receivers: [otlp]
-    #  processors: [batch]
-    #  exporters: [logging, otlp, jaeger]
+#    traces:
+#      receivers: [otlp]
+#      processors: [batch]
+#      exporters: [otlp]
+
     metrics:
-      receivers: [otlp, hostmetrics]
+      receivers: [prometheus]
       processors: [batch]
-      exporters: 
-         - prometheus
-         {{- if eq .otelDefaults.prometheusRemoteWrite.enable "true" }}
-         - prometheusremotewrite
-         {{- end }}
-#      exporters: [logging, prometheus{{ if eq .otelDefaults.prometheusRemoteWrite.enable "true" }}, prometheusremotewrite{{ end }}]
+      exporters:
+        - prometheus
+        {{- if eq .otelDefaults.prometheusRemoteWrite.enable "true" }}
+        - prometheusremotewrite
+        {{- end }}
diff --git a/charts/hedera-network/templates/services/network-node-svc.yaml b/charts/hedera-network/templates/services/network-node-svc.yaml
@@ -11,23 +11,24 @@ spec:
   selector:
     app: network-{{ $nodeConfig.name }}
   ports:
+  # GOSSIP port is used by nodes for gossip protocol
   - name: gossip
     protocol: TCP
     port: 50111 # gossip port
     targetPort: 50111
+  # GRPC-TLS port is used for services API
   - name: grpc-non-tls
     protocol: TCP
     port: 50211 # non-tls grpc client port
     targetPort: 50211
+  # GRPC-TLS port
   - name: grpc-tls
     protocol: TCP
     port: 50212 # tls grpc client port
     targetPort: 50212
-  - name: otel-metrics
-    protocol: TCP
-    port: 8888
-    targetPort: 8888
-  - name: prometheus
+  # Node metrics port
+  # Rather than exposing the node's metrics port 9999 directly, we expose otel-collect's prometheus port here.
+  - name: prometheus # otel-collector's prometheus exporter port
     protocol: TCP
     port: 8889
     targetPort: 8889

diff --git a/charts/hedera-network/templates/sidecars/_otel-collector.tpl b/charts/hedera-network/templates/sidecars/_otel-collector.tpl
@@ -8,26 +8,26 @@
   securityContext:
     {{- include "fullstack.root.security.context" . | nindent 4 }}
   ports:
-    - name: otel-health
-      containerPort: 13133
+    - name: otlp
+      containerPort: 4317 # otel port defined in otel-collector config
       protocol: TCP
-    - name: otel-metrics
-      containerPort: 8888
+    - name: prometheus # prometheus exporter port as specified in otel-collector-config.yaml
+      containerPort: 8889
       protocol: TCP
-    - name: otel-otlp
-      containerPort: 4317
+    - name: health
+      containerPort: 13133
       protocol: TCP
-    - name: prometheus
-      containerPort: 8889
+    - name: metrics # default metrics port exposed by the otel-collector itself
+      containerPort: 8888
       protocol: TCP
-  {{- with default $defaults.livenessProbe $otel.livenessProbe }}
   livenessProbe:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-  {{- with default $defaults.readinessProbe $otel.readinessProbe }}
+    httpGet:
+      path: /
+      port: health
   readinessProbe:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
+    httpGet:
+      path: /
+      port: health
   volumeMounts:
     - name: otel-collector-volume
       mountPath: /etc/otelcol-contrib/config.yaml

diff --git a/charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml b/charts/hedera-network/templates/telemetry/prometheus-svc-monitor.yaml
@@ -10,8 +10,6 @@ spec:
     matchLabels:
       fullstack.hedera.com/type: network-node-svc
   endpoints:
-    - port: otel-metrics
-      interval: 5s
-    - port: prometheus
-      interval: 5s
+    - port: prometheus # must match the prometheus port-name in network-node-svc.yaml
+      interval: 10s # ideally it should be higher than the node-metrics-scraper interval set in otel-collector-config.yaml
 {{- end }}
diff --git a/charts/hedera-network/values.yaml b/charts/hedera-network/values.yaml
@@ -163,14 +163,6 @@ defaults:
         repository: "otel/opentelemetry-collector-contrib"
         tag: "0.72.0"
         pullPolicy: "IfNotPresent"
-      livenessProbe:
-        httpGet:
-          path: /
-          port: otel-health
-      readinessProbe:
-        httpGet:
-          path: /
-          port: otel-health
       resources: {}
       prometheusRemoteWrite:
         enable: "false"

diff --git a/dev/telemetry/prometheus/example-app.yaml b/dev/telemetry/prometheus/example-app.yaml
@@ -16,19 +16,19 @@ spec:
         - name: prometheus-example-app
           image: fabxc/instrumented_app
           ports:
-            - name: otel-metrics # use same name as used by otel-collector sidecar in hedera-network network-node
+            - name: app-port
               containerPort: 8080
 ---
 kind: Service
 apiVersion: v1
 metadata:
   name: prometheus-example-app
   labels:
-    # use the same label used by hedera-network network-node-svc
+    # use the same label used in network-node-svc.yaml
     fullstack.hedera.com/type: network-node-svc
 spec:
   selector:
     app: prometheus-example-app
   ports:
-    - name: otel-metrics
+    - name: prometheus # use same prometheus port-name as in network-node-svc.yaml
       port: 8080