After deploy prometheus, it shows x509: certificate is valid for apiserver, not kubernetes.default.svc

[zybjcdl]

I deploy prometheus on a k8s cluster, but it shows
kubernetes-cluster Endpoint State Labels Last Scrape Error
https://kubernetes.default.svc:443/metrics DOWN none 3.153s ago Get https://kubernetes.default.svc:443/metrics: x509: certificate is valid for apiserver, not kubernetes.default.svc

And also, no kubernetes-nodes shows up.

Here is my prometheus.yml:

---
apiVersion: v1
kind: Service
metadata:
  annotations:
    prometheus.io/scrape: 'true'
  labels:
    name: prometheus
  name: prometheus
  namespace: monitoring
spec:
  selector:
    app: prometheus
  ports:
  - port: 80
    targetPort: 9090

---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    name: prometheus-deployment
  name: prometheus
  namespace: monitoring
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      containers:
      - image: quay.io/prometheus/prometheus:v1.0.1
        name: prometheus
        command:
        - "/bin/prometheus"
        args:
        - "-config.file=/etc/prometheus/prometheus.yml"
        - "-storage.local.path=/prometheus"
        - "-storage.local.retention=24h"
        - "-web.external-url=http://localhost:8080/api/v1/proxy/namespaces/monitoring/services/prometheus/"
        - "-web.route-prefix=/"
        - "-alertmanager.url=http://localhost:8080/api/v1/proxy/namespaces/monitoring/services/alertmanager/"
        ports:
        - containerPort: 9090
          protocol: TCP
        volumeMounts:
        - mountPath: "/prometheus"
          name: data
        - mountPath: "/etc/prometheus"
          name: config-volume
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            cpu: 500m
            memory: 2500Mi
      volumes:
      - emptyDir: {}
        name: data
      - configMap:
          name: prometheus-config
        name: config-volume

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitoring
data:
  # alert.rules
  alert.rules: |
    # Alert for any instance that is unreachable for >5 minutes.
    ALERT InstanceUp
      IF up == 1
      LABELS { job = "kubernetes-nodes" }
      ANNOTATIONS {
        summary = "Instance {{ $labels.instance }} Up",
        description = "{{ $labels.instance }} of job {{ $labels.job }} has been up.",
      }
  # prometheus.yml
  prometheus.yml: |
    global:
      scrape_interval: 30s
      scrape_timeout: 30s
    rule_files:
      - '/etc/prometheus/alert.rules'
    scrape_configs:
    - job_name: 'prometheus'
      static_configs:
        - targets: ['localhost:9090']
    - job_name: 'kubernetes-cluster'
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - api_servers:
        - 'https://kubernetes.default.svc'
        in_cluster: true
        role: apiserver
    - job_name: 'kubernetes-nodes'
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        insecure_skip_verify: true
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - api_servers:
        - 'https://kubernetes.default.svc'
        in_cluster: true
        role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
    - job_name: 'kubernetes-service-endpoints'
      scheme: https
      kubernetes_sd_configs:
      - api_servers:
        - 'https://kubernetes.default.svc'
        in_cluster: true
        role: endpoint
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)(?::\d+);(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_service_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name
    - job_name: 'kubernetes-services'
      scheme: https
      metrics_path: /probe
      params:
        module: [http_2xx]
      kubernetes_sd_configs:
      - api_servers:
        - 'https://kubernetes.default.svc'
        in_cluster: true
        role: service
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
        action: keep
        regex: true
      - source_labels: [__address__]
        target_label: __param_target
      - target_label: __address__
        replacement: blackbox
      - source_labels: [__param_target]
        target_label: instance
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_service_namespace]
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        target_label: kubernetes_name
    - job_name: 'kubernetes-pods'
      scheme: https
      kubernetes_sd_configs:
      - api_servers:
        - 'https://kubernetes.default.svc'
        in_cluster: true
        role: pod
      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: (.+):(?:\d+);(\d+)
        replacement: ${1}:${2}
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_pod_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name

[zybjcdl]

BTW:
kubectl version:
Client Version: version.Info{Major:"1", Minor:"3+", GitVersion:"v1.3.6-alpha-dirty", GitCommit:"eb504d96229a6c98cbb4d26cdfb7be25c148e259", GitTreeState:"dirty", BuildDate:"2016-09-14T04:57:23Z", GoVersion:"go1.6.2", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"3+", GitVersion:"v1.3.6-alpha-dirty", GitCommit:"eb504d96229a6c98cbb4d26cdfb7be25c148e259", GitTreeState:"dirty", BuildDate:"2016-09-14T04:51:57Z", GoVersion:"go1.6.2", Compiler:"gc", Platform:"linux/amd64"}

[jimmidyson]

Your certificates doesn't contain the default CN or SAN (recommended by kubernetes) of kubernetes.default.svc so default config can't validate the server certificate. Some options:

1. Reissue your API server certificate with the additional CN.
2. Change the api server address in Prometheus config (note that the host name will still need to match a CN or SAN in the certificate to validate).
3. Disable TLS validation in Prometheus config. This is not recommended but is the only way to work with self signed certs.

2 is probably your best bet I think.

[zybjcdl]

@jimmidyson Thank you for your response.
Could you please be more specific how could I change the config file?

My kubernetes dashboard is https://9.30.245.34/ui , the private ip is 172.20.154.140
When I access my dashboard ,I need to enter admin/passwd

How could I change the config file to access this api?

I tried to change the config as below:
- job_name: 'kubernetes-cluster'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- api_servers:
- 'https://172.20.154.140'
basic_auth:
username: admin
password: passwd
in_cluster: true
role: apiserver

But it reports:
server returned HTTP status 403 Forbidden

And also, I checked, in prometheus container, the /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
is the same as /etc/kubernetes/cert/cal.pem
But it is different from /etc/kubernetes/cert/apiserver.pem

[jimmidyson]

Could you format the config in the issue by wrapping in ``` before & after the contents? YAML is picky on indentation so without that formatting I can't see what you might have got wrong.

[zybjcdl]

@jimmidyson Here is the before & after.
I also marked the changed part with bold characters
Before
- job_name: 'kubernetes-cluster'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- api_servers:
- 'https://kubernetes.default.svc'
in_cluster: true
role: apiserver

After:
- job_name: 'kubernetes-cluster'
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
insecure_skip_verify: true
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
kubernetes_sd_configs:
- api_servers:
- 'https://172.20.154.140'
basic_auth:
username: admin
password: passwd
in_cluster: true
role: apiserver

[jimmidyson]

Again, can you please format your config by wrapping in ```?

It looks OK so I've had a look in the retrieval code & it looks like in_cluster config option will enable bearer token auth in preference to the basic auth config you've added (they both use the same Authorization header so only one can be used at a time).

To work around, try removing the in_cluster option & set tlsConfig.CAFile as you've done above in SD config section, something like:

kubernetes_sd_configs:
  - api_servers:
    - 'https://172.20.154.140'
  role: apiserver
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  basic_auth:
    username: admin
    password: passwd

[zybjcdl]

@jimmidyson
I tried your suggestion, now ,it shows "Get https://172.20.154.140:443/metrics: x509: certificate signed by unknown authority"

here is my config:

  prometheus.yml: |
    global:
      scrape_interval: 30s
      scrape_timeout: 30s
    rule_files:
      - '/etc/prometheus/alert.rules'
    scrape_configs:
    - job_name: 'prometheus'
      static_configs:
        - targets: ['localhost:9090']
    - job_name: 'kubernetes-cluster'
      scheme: https
      #tls_config:
      #  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      #  insecure_skip_verify: true
      #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - api_servers:
        - 'https://172.20.154.140'
        #- 'https://kubernetes.default.svc'
        in_cluster: false
        role: apiserver
        tls_config:
          ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        basic_auth:
          username: admin
          password: passwd

Also, I checked ,my apiserver is running with following parameters:

/hyperkube apiserver --bind-address=0.0.0.0 --insecure-bind-address=127.0.0.1 --etcd-servers=http://172.20.154.140:4001 --allow-privileged=true --service-cluster-ip-range=10.10.10.0/24 --secure-port=8443 --insecure-port=8080 --advertise-address=172.20.154.140 --tls-cert-file=/etc/kubernetes/cert/apiserver.pem --tls-private-key-file=/etc/kubernetes/cert/apiserver-key.pem --client-ca-file=/etc/kubernetes/cert/ca.pem --authorization-mode=Webhook --authorization-webhook-config-file=/etc/kubernetes/webhook-config --admission-control=LimitRanger,NamespaceLifecycle,ServiceAccount,ResourceQuota --basic-auth-file=/etc/kubernetes/baseauth.csv --v=2

In prometheus container, the /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
is the same as /etc/kubernetes/cert/cal.pem
But it is different from /etc/kubernetes/cert/apiserver.pem

[zybjcdl]

@jimmidyson
Finally, I succesfully scrape kubernetes-cluster target.
Here is my config:

 - job_name: 'kubernetes-cluster'
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      basic_auth:
        username: admin
        password: passwd
      kubernetes_sd_configs:
      - api_servers:
        - 'https://172.20.154.140'
        in_cluster: true
        role: apiserver

I still have 2 questions:

1. You can see I put tls_config & basic_auth in job config, instead of kubernetes_sd_config.
   I still use in_cluster: true. And that worked.
   So I am a little confused ,what are the difference between scraping and discovering?

2. You can see I could successfully scrape kubernetes-cluster, but only kubernetes-cluster.
   For the other job, I couldn't see any item in the console.
   I am using the same auth in the config, I don't know why.
   Checking the log, it shows:
   time="2016-10-20T07:51:09Z" level=error msg="Cannot initialize services collection: unable to list Kubernetes services; unexpected response: 403 403 Forbidden" source="service.go:117"
   time="2016-10-20T07:51:09Z" level=error msg="Cannot initialize nodes collection: unable to list Kubernetes nodes; unexpected response: 403 403 Forbidden" source="node.go:124"

Here is my whole config:

  prometheus.yml: |
    global:
      scrape_interval: 30s
      scrape_timeout: 30s
    rule_files:
      - '/etc/prometheus/alert.rules'
    scrape_configs:
    - job_name: 'prometheus'
      static_configs:
        - targets: ['localhost:9090']
    - job_name: 'kubernetes-nodes'
      scheme: http
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      basic_auth:
        username: admin
        password: passwd
      kubernetes_sd_configs:
      - api_servers:
        - 'https://172.20.154.140'
        in_cluster: true
        role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
    - job_name: 'kubernetes-cluster'
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      basic_auth:
        username: admin
        password: passwd
      kubernetes_sd_configs:
      - api_servers:
        - 'https://172.20.154.140'
        in_cluster: true
        role: apiserver
    - job_name: 'kubernetes-service-endpoints'
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      basic_auth:
        username: admin
        password: passwd
      kubernetes_sd_configs:
      - api_servers:
        - 'https://172.20.154.140'
        in_cluster: true
        role: endpoint
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)(?::\d+);(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_service_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name
    - job_name: 'kubernetes-services'
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      basic_auth:
        username: admin
        password: passwd
      metrics_path: /probe
      params:
        module: [http_2xx]
      kubernetes_sd_configs:
      - api_servers:
        - 'https://172.20.154.140'
        in_cluster: true
        role: service
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
        action: keep
        regex: true
      - source_labels: [__address__]
        target_label: __param_target
      - target_label: __address__
        replacement: blackbox
      - source_labels: [__param_target]
        target_label: instance
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_service_namespace]
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        target_label: kubernetes_name
    - job_name: 'kubernetes-pods'
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      basic_auth:
        username: admin
        password: passwd
      kubernetes_sd_configs:
      - api_servers:
        - 'https://172.20.154.140'
        in_cluster: true
        role: pod
      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: (.+):(?:\d+);(\d+)
        replacement: ${1}:${2}
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_pod_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name

[brian-brazil]

I'm presuming this is addressed by now.

[ghost]

i am facing same issue,, my prometheus is outside of k8s cluster..
i can telnet to port 443 from the prometheus server to the k8s master
Can any onel tell me from where can i get the ca_file from and i am using username & password from the machine i managing my k8s cluster from ,,its in aws

 - job_name: 'kubernetes-api'
   static_configs:
   - targets: ['api.internal.domain.net:443']
   kubernetes_sd_configs:
   - role: endpoints
     api_server: https://api.internal.domain.net
   scheme: https
   relabel_configs:
   - action: keep
     regex: default;kubernetes;https
     source_labels:
     - __meta_kubernetes_namespace
     - __meta_kubernetes_service_name
     - __meta_kubernetes_endpoint_port_name
   tls_config:
     ca_file: /etc/prometheus/ca.crt
     insecure_skip_verify: true
   basic_auth:
     username: admin
     password: password

[strongit]

i am facing same issue,, my prometheus is outside of k8s cluster..
i can telnet to port 443 from the prometheus server to the k8s master
Can any onel tell me from where can i get the ca_file from and i am using username & password from the machine i managing my k8s cluster from ,,its in aws

 - job_name: 'kubernetes-api'
   static_configs:
   - targets: ['api.internal.domain.net:443']
   kubernetes_sd_configs:
   - role: endpoints
     api_server: https://api.internal.domain.net
   scheme: https
   relabel_configs:
   - action: keep
     regex: default;kubernetes;https
     source_labels:
     - __meta_kubernetes_namespace
     - __meta_kubernetes_service_name
     - __meta_kubernetes_endpoint_port_name
   tls_config:
     ca_file: /etc/prometheus/ca.crt
     insecure_skip_verify: true
   basic_auth:
     username: admin
     password: password

my prometheus is outside of k8s cluster too,and ,it shows msg="prometheus/discovery/kubernetes/kubernetes.go:372: Failed to list *v1.Node: Get https://10.202.183.187:16443/api/v1/nodes?limit=500&resourceVersion=0: x509: certificate signed by unknown authority"
config show below:
`global:
scrape_interval: 30s
scrape_timeout: 30s
rule_files:

• '/etc/prometheus/alert.rules'

remote_write:

• url: "http://10.203.14.13:8086/api/v1/prom/write?db=prom_test"
  remote_read:
• url: "http://10.203.14.13:8086/api/v1/prom/read?db=prom_test"

scrape_configs:

• job_name: 'prometheus'
  static_configs:

  • targets: ['localhost:9090']

• job_name: 'kubernetes-nodes'
  scheme: http
  tls_config:
  insecure_skip_verify: true
  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  basic_auth:
  username: admin
  password: password
  kubernetes_sd_configs:

  • api_server: 'https://10.202.183.187:16443'
    role: node
    relabel_configs:
  • action: labelmap
    regex: _meta_kubernetes_node_label(.+)

• job_name: 'kubernetes-service-endpoints'
  scheme: https
  tls_config:
  insecure_skip_verify: true
  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  basic_auth:
  username: admin
  password: password
  kubernetes_sd_configs:

  • api_server: 'https://10.202.183.187:16443'
    role: endpoints
    relabel_configs:
  • source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
    action: keep
    regex: true
  • source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
    action: replace
    target_label: scheme
    regex: (https?)
  • source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
    action: replace
    target_label: metrics_path
    regex: (.+)
  • source_labels: [address, __meta_kubernetes_service_annotation_prometheus_io_port]
    action: replace
    target_label: address
    regex: (.+)(?::\d+);(\d+)
    replacement: $1:$2
  • action: labelmap
    regex: _meta_kubernetes_service_label(.+)
  • source_labels: [__meta_kubernetes_service_namespace]
    action: replace
    target_label: kubernetes_namespace
  • source_labels: [__meta_kubernetes_service_name]
    action: replace
    target_label: kubernetes_name

• job_name: 'kubernetes-services'
  scheme: https
  tls_config:
  insecure_skip_verify: true
  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  basic_auth:
  username: admin
  password: password
  metrics_path: /probe
  params:
  module: [http_2xx]
  kubernetes_sd_configs:

  • api_server: 'https://10.202.183.187:16443'
    role: service
    relabel_configs:
  • source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
    action: keep
    regex: true
  • source_labels: [address]
    target_label: __param_target
  • target_label: address
    replacement: blackbox
  • source_labels: [__param_target]
    target_label: instance
  • action: labelmap
    regex: _meta_kubernetes_service_label(.+)
  • source_labels: [__meta_kubernetes_service_namespace]
    target_label: kubernetes_namespace
  • source_labels: [__meta_kubernetes_service_name]
    target_label: kubernetes_name

• job_name: 'kubernetes-pods'
  scheme: https
  tls_config:
  insecure_skip_verify: true
  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  basic_auth:
  username: admin
  password: password
  kubernetes_sd_configs:

  • api_server: 'https://10.202.183.187:16443'
    role: pod
    relabel_configs:
  • source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
    action: keep
    regex: true
  • source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
    action: replace
    target_label: metrics_path
    regex: (.+)
  • source_labels: [address, __meta_kubernetes_pod_annotation_prometheus_io_port]
    action: replace
    regex: (.+):(?:\d+);(\d+)
    replacement: ${1}:${2}
    target_label: address
  • action: labelmap
    regex: _meta_kubernetes_pod_label(.+)
  • source_labels: [__meta_kubernetes_pod_namespace]
    action: replace
    target_label: kubernetes_namespace
  • source_labels: [__meta_kubernetes_pod_name]
    action: replace
    target_label: kubernetes_pod_name`