Expose metrics to my local kube-prometheus.

[BXJC]

Write down your inquiry

I'm currently working with my local Minikube environment and encountering difficulties in retrieving metrics from my application. I've gone through the instructions provided in Integrate Online Boutique with Google Cloud Operations. Could someone please guide me on how to expose metrics, traces, and logs to my local cluster? Your assistance would be greatly appreciated!
I revised otel-collector.yaml and kustomization.yaml like these below:

otel-collector.yaml

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: opentelemetrycollector
spec:
  replicas: 1
  selector:
    matchLabels:
      app: opentelemetrycollector
  template:
    metadata:
      labels:
        app: opentelemetrycollector
    spec:
      securityContext:
        fsGroup: 1000
        runAsGroup: 1000
        runAsNonRoot: true
        runAsUser: 1000
      # Init container retrieves the current cloud project id from the metadata server
      # and inserts it into the collector config template
      # https://cloud.google.com/compute/docs/storing-retrieving-metadata
      initContainers:
      - name: otel-gateway-init
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
              - ALL
          privileged: false
          readOnlyRootFilesystem: true
        image: busybox:latest@sha256:c3839dd800b9eb7603340509769c43e146a74c63dca3045a8e7dc8ee07e53966
        command:
        - '/bin/sh'
        - '-c'
        - cp /template/collector-gateway-config-template.yaml /conf/collector-gateway-config.yaml
          # sed "s/{{PROJECT_ID}}/$(curl -H 'Metadata-Flavor: Google' http://metadata.google.internal/computeMetadata/v1/project/project-id)/" /template/collector-gateway-config-template.yaml >> /conf/collector-gateway-config.yaml
        volumeMounts:
        - name: collector-gateway-config-template
          mountPath: /template
        - name: collector-gateway-config
          mountPath: /conf
      containers:
      # This gateway container will receive traces and metrics from each microservice
      # and forward it to GCP
      - name: otel-gateway
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
              - ALL
          privileged: false
          readOnlyRootFilesystem: true
        args:
        - --config=/conf/collector-gateway-config.yaml
        image: otel/opentelemetry-collector-contrib:0.98.0@sha256:5cea85bcbc734a3c0a641368e5a4ea9d31b472997e9f2feca57eeb4a147fcf1a
        volumeMounts:
        - name: collector-gateway-config
          mountPath: /conf
      volumes:
      # Simple ConfigMap volume with template file
      - name: collector-gateway-config-template
        configMap:
          items:
          - key: collector-gateway-config-template.yaml
            path: collector-gateway-config-template.yaml
          name: collector-gateway-config-template
      # Create a volume to store the expanded template (with correct cloud project ID)
      - name: collector-gateway-config
        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  name: opentelemetrycollector
spec:
  ports:
  - name: grpc-otlp
    port: 4317
    protocol: TCP
    targetPort: 4317
  selector:
    app: opentelemetrycollector
  type: ClusterIP
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: collector-gateway-config-template
# Open Telemetry Collector config
# https://opentelemetry.io/docs/collector/configuration/
data:
  collector-gateway-config-template.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
    processors:

    exporters:
      otlp:
        endpoint: localhost:4317


      prometheus:
        endpoint: localhost:9090
        namespace: monitoring

    extensions:
      health_check:
      pprof:
      zpages:

    service:
      extensions: [health_check, pprof, zpages]
      pipelines:
        # traces:
        #   receivers: [otlp]
        #   processors: []
        #   exporters: [otlp]
        metrics:
          receivers: [otlp]
          processors: []
          exporters: [prometheus]
        # logs:
        #   receivers: [otlp]
        #   processors: []
        #   exporters: [otlp]

    # receivers:
    #   otlp:
    #     protocols: 
    #       grpc:
    # processors:
    # exporters:
    #   googlecloud:
    #     project: {{PROJECT_ID}}
    # service:
    #   pipelines:
    #     traces:
    #       receivers: [otlp] # Receive otlp-formatted data from other collector instances
    #       processors: []
    #       exporters: [googlecloud] # Export traces directly to Google Cloud
    #     metrics:
    #       receivers: [otlp]
    #       processors: []
    #       exporters: [googlecloud] # Export metrics to Google Cloud

kustomization.yaml

apiVersion: kustomize.config.k8s.io/v1alpha1
kind: Component
resources:
  - otel-collector.yaml
patches:
# adservice - not yet implemented
# checkoutservice - tracing, profiler
- patch: |-
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: checkoutservice
    spec:
      template:
        spec:
          containers:
            - name: server
              env:
              - name: COLLECTOR_SERVICE_ADDR
                value: "opentelemetrycollector:4317"
              - name: OTEL_SERVICE_NAME
                value: "checkoutservice"
              - name: ENABLE_TRACING
                value: "1"
              # - name: ENABLE_PROFILER
              #   value: "1"
# currencyservice - tracing, profiler
- patch: |-
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: currencyservice
    spec:
      template:
        spec:
          containers:
            - name: server
              env:
              - name: COLLECTOR_SERVICE_ADDR
                value: "opentelemetrycollector:4317"
              - name: OTEL_SERVICE_NAME
                value: "currencyservice"
              - name: ENABLE_TRACING
                value: "1"
              # - name: DISABLE_PROFILER
              #   $patch: delete
# emailservice - tracing, profiler
- patch: |-
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: emailservice
    spec:
      template:
        spec:
          containers:
            - name: server
              env:
              - name: COLLECTOR_SERVICE_ADDR
                value: "opentelemetrycollector:4317"
              - name: OTEL_SERVICE_NAME
                value: "emailservice"
              - name: ENABLE_TRACING
                value: "1"
              # - name: DISABLE_PROFILER
              #   $patch: delete
# frontend - tracing, profiler
- patch: |-
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: frontend
    spec:
      template:
        spec:
          containers:
            - name: server
              env:
              - name: ENABLE_TRACING
                value: "1"
              - name: COLLECTOR_SERVICE_ADDR
                value: "opentelemetrycollector:4317"
              - name: OTEL_SERVICE_NAME
                value: "frontend"
              # - name: ENABLE_PROFILER
              #   value: "1"
# paymentservice - tracing, profiler
- patch: |-
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: paymentservice
    spec:
      template:
        spec:
          containers:
            - name: server
              env:
              - name: COLLECTOR_SERVICE_ADDR
                value: "opentelemetrycollector:4317"
              - name: OTEL_SERVICE_NAME
                value: "paymentservice"
              - name: ENABLE_TRACING
                value: "1"
              # - name: DISABLE_PROFILER
              #   $patch: delete
# productcatalogservice - tracing, profiler
- patch: |-
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: productcatalogservice
    spec:
      template:
        spec:
          containers:
            - name: server
              env:
              - name: COLLECTOR_SERVICE_ADDR
                value: "opentelemetrycollector:4317"
              - name: OTEL_SERVICE_NAME
                value: "productcatalogservice"
              - name: ENABLE_TRACING
                value: "1"
              # - name: DISABLE_PROFILER
              #   value: "1"
# recommendationservice - tracing, profiler
- patch: |-
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: recommendationservice
    spec:
      template:
        spec:
          containers:
            - name: server
              env:
              - name: COLLECTOR_SERVICE_ADDR
                value: "opentelemetrycollector:4317"
              - name: OTEL_SERVICE_NAME
                value: "recommendationservice"
              - name: ENABLE_TRACING
                value: "1"
              # - name: DISABLE_PROFILER
              #   $patch: delete
# shippingservice - stats, tracing, profiler
- patch: |-
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: shippingservice
    spec:
      template:
        spec:
          containers:
            - name: server
              env:
              # - name: DISABLE_PROFILER
              #   $patch: delete

kube-prometheus was configured in default in monitoring namespace.