Problem with config.load_incluster_config()

[Michalski-Piotr]

Hello,
On Kubernetes cluster version 1.25.4 I'm testing the Python kubernetes client 26.1 using test script from https://github.com/kubernetes-client/python/blob/master/examples/in_cluster_config.py

[oracle@nm1 kubernetes]$ pwd
/u01/oracle/.local/lib/python3.6/site-packages/kubernetes
[oracle@nm1 kubernetes]$ cat __init__.py
# Copyright 2022 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__project__ = 'kubernetes'
# The version is auto-updated. Please do not edit.
__version__ = "26.1.0"

from . import client
from . import config
from . import dynamic
from . import watch
from . import stream
from . import utils
from . import leaderelection

Test:

$ cat test.py
from kubernetes import client, config


def main():
    config.load_incluster_config()

    v1 = client.CoreV1Api()
    print("Listing pods with their IPs:")
    ret = v1.list_pod_for_all_namespaces(watch=False)
    for i in ret.items:
        print("%s\t%s\t%s" %
              (i.status.pod_ip, i.metadata.namespace, i.metadata.name))


if __name__ == '__main__':
    main()
    

[oracle@nm1 ~]$ python3 test.py
Listing pods with their IPs:
Traceback (most recent call last):
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/connection.py", line 175, in _new_conn
    (self._dns_host, self.port), self.timeout, **extra_kw
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/util/connection.py", line 95, in create_connection
    raise err
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/util/connection.py", line 85, in create_connection
    sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 710, in urlopen
    chunked=chunked,
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 386, in _make_request
    self._validate_conn(conn)
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 1042, in _validate_conn
    conn.connect()
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/connection.py", line 363, in connect
    self.sock = conn = self._new_conn()
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/connection.py", line 187, in _new_conn
    self, "Failed to establish a new connection: %s" % e
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7f335a9bce80>: Failed to establish a new connection: [Errno 111] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "test.py", line 16, in <module>
    main()
  File "test.py", line 9, in main
    ret = v1.list_pod_for_all_namespaces(watch=False)
  File "/u01/oracle/.local/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py", line 17309, in list_pod_for_all_namespaces
    return self.list_pod_for_all_namespaces_with_http_info(**kwargs)  # noqa: E501
  File "/u01/oracle/.local/lib/python3.6/site-packages/kubernetes/client/api/core_v1_api.py", line 17430, in list_pod_for_all_namespaces_with_http_info
    collection_formats=collection_formats)
  File "/u01/oracle/.local/lib/python3.6/site-packages/kubernetes/client/api_client.py", line 353, in call_api
    _preload_content, _request_timeout, _host)
  File "/u01/oracle/.local/lib/python3.6/site-packages/kubernetes/client/api_client.py", line 184, in __call_api
    _request_timeout=_request_timeout)
  File "/u01/oracle/.local/lib/python3.6/site-packages/kubernetes/client/api_client.py", line 377, in request
    headers=headers)
  File "/u01/oracle/.local/lib/python3.6/site-packages/kubernetes/client/rest.py", line 245, in GET
    query_params=query_params)
  File "/u01/oracle/.local/lib/python3.6/site-packages/kubernetes/client/rest.py", line 218, in request
    headers=headers)
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/request.py", line 75, in request
    method, url, fields=fields, headers=headers, **urlopen_kw
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/request.py", line 96, in request_encode_url
    return self.urlopen(method, url, **extra_kw)
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/poolmanager.py", line 376, in urlopen
    response = conn.urlopen(method, u.request_uri, **kw)
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 828, in urlopen
    **response_kw
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 828, in urlopen
    **response_kw
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 828, in urlopen
    **response_kw
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 788, in urlopen
    method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
  File "/u01/oracle/.local/lib/python3.6/site-packages/urllib3/util/retry.py", line 592, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='10.28.0.1', port=443): Max retries exceeded with url: /api/v1/pods?watch=False (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7f335a9bce80>: Failed to establish a new connection: [Errno 111] Connection refused',))

It looks there are some additional requirements. Could you help to understand what is missing, please?

I also did a manual test as below:

(env_tools) [opc@k8s-registry FMW-K8s]$ python3.6
Python 3.6.8 (default, Nov 18 2021, 10:07:16)
[GCC 4.8.5 20150623 (Red Hat 4.8.5-44.0.3)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>>
>>> import sys
>>> import time
>>> import logging
>>> import argparse
>>> import re
>>> from subprocess import call
>>> from kubernetes import client, watch, config

>>> from kubernetes.stream import stream
>>> v1 = config.load_incluster_config()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/mnt/oke_shared/repos/FMW-K8s/tools/bin/env_tools/lib64/python3.6/site-packages/kubernetes/config/incluster_config.py", line 121, in load_incluster_config
    try_refresh_token=try_refresh_token).load_and_set(client_configuration)
  File "/mnt/oke_shared/repos/FMW-K8s/tools/bin/env_tools/lib64/python3.6/site-packages/kubernetes/config/incluster_config.py", line 54, in load_and_set
    self._load_config()
  File "/mnt/oke_shared/repos/FMW-K8s/tools/bin/env_tools/lib64/python3.6/site-packages/kubernetes/config/incluster_config.py", line 62, in _load_config
    raise ConfigException("Service host/port is not set.")
kubernetes.config.config_exception.ConfigException: Service host/port is not set.
>>>

Thank you in advance for any help in this issue.

In the past I was testing the same on Kubernetes 1.23 version and it was working fine.

Regards,
Piotr Michalski

Environment:

• Kubernetes version (kubectl version): 1.25.4
• OS (e.g., MacOS 10.13.6): RHEL 7.9
• Python version (python --version): 3.6
• Python client version (pip list | grep kubernetes) 26.1

[roycaihw]

/help

[Michalski-Piotr]

Hello,
I reviewed the update, but it is not clear for me, what additional data I should provide.
I'm observing this issue from time to to time. If you need any additional information, I'm more than happy to deliver.
Please post the requirements as update to this issue.
Thank you in advance.
Regards,
Piotr Michalski

[Michalski-Piotr]

/remove-help

[tomplus]

Hi Piotr. You use the example which works only within cluster. It won't work on your local environment. There is a comment how to run it: https://github.com/kubernetes-client/python/blob/master/examples/in_cluster_config.py#L16

[Michalski-Piotr]

Hello @tomplus,
thank you for this example. I was testing provided example inside the pod running in the cluster. I will test your example and provide the feedback here. Currently in the same environment I have issues with flannel/coredns, so environment is out of service.
Regards,
Piotr

[Michalski-Piotr]

Hello @tomplus,
I reprovisioned Oracle OCI cluster 1.25.4 from scratch and I'm still getting the error using the example code.

$ cat in_cluster_config.py
from kubernetes import client, config


def main():
    config.load_incluster_config()

    v1 = client.CoreV1Api()
    print("Listing pods with their IPs:")
    ret = v1.list_pod_for_all_namespaces(watch=False)
    for i in ret.items:
        print("%s\t%s\t%s" %
              (i.status.pod_ip, i.metadata.namespace, i.metadata.name))

if __name__ == '__main__':
    main()

execution:

$ python3 in_cluster_config.py
Traceback (most recent call last):
  File "in_cluster_config.py", line 16, in <module>
    main()
  File "in_cluster_config.py", line 5, in main
    config.load_incluster_config()
  File "/mnt/oke_shared/repos/FMW-K8s/tools/bin/env_tools/lib64/python3.6/site-packages/kubernetes/config/incluster_config.py", line 121, in load_incluster_config
    try_refresh_token=try_refresh_token).load_and_set(client_configuration)
  File "/mnt/oke_shared/repos/FMW-K8s/tools/bin/env_tools/lib64/python3.6/site-packages/kubernetes/config/incluster_config.py", line 54, in load_and_set
    self._load_config()
  File "/mnt/oke_shared/repos/FMW-K8s/tools/bin/env_tools/lib64/python3.6/site-packages/kubernetes/config/incluster_config.py", line 62, in _load_config
    raise ConfigException("Service host/port is not set.")
kubernetes.config.config_exception.ConfigException: Service host/port is not set.

Regards,
Piotr

[Michalski-Piotr]

Hello,
similar issue was raised here #992

In my case I applied ClusterRole and ClusterRoleBinding according to the example

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: pods-list
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: pods-list
subjects:
- kind: ServiceAccount
  name: default
  namespace: default
roleRef:
  kind: ClusterRole
  name: pods-list
  apiGroup: rbac.authorization.k8s.io

Regards,
Piotr

[tomplus]

Are you inside your pod? For some reason environment variables KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT are not set. Could you try to call printenv to see what variables are set.

[Michalski-Piotr]

Hello @tomplus,
thank you. You are right, my bad. From pod, in the new cluster all is working fine:

[opc@k8s-registry kubernetes]$ k get pods
NAME                            READY   STATUS    RESTARTS      AGE
busybox                         1/1     Running   3 (18m ago)   3h18m
hello-python-5dc999f598-qkzj6   1/1     Running   0             3m57s
hello-python-5dc999f598-xwdl7   1/1     Running   0             3m56s
[opc@k8s-registry kubernetes]$

$ k exec -it hello-python-5dc999f598-qkzj6 -- /bin/bash -l

cat test.py
from kubernetes import client, config


def main():
    config.load_incluster_config()

    v1 = client.CoreV1Api()
    print("Listing pods with their IPs:")
    ret = v1.list_pod_for_all_namespaces(watch=False)
    for i in ret.items:
        print("%s\t%s\t%s" %
              (i.status.pod_ip, i.metadata.namespace, i.metadata.name))

if __name__ == '__main__':
    main()

root@hello-python-5dc999f598-qkzj6:/# python3.7 test.py
Listing pods with their IPs:
10.27.0.130	default	busybox
10.27.0.139	default	hello-python-5dc999f598-qkzj6
10.27.0.13	default	hello-python-5dc999f598-xwdl7
10.27.0.4	ingress-nginx	ingress-nginx-admission-create-2x24d
10.27.0.131	ingress-nginx	ingress-nginx-admission-patch-5xcbk
10.27.0.5	ingress-nginx	ingress-nginx-controller-8574b6d7c9-hcp6g
None	kube-system	coredns-8488499cff-76kgd
None	kube-system	coredns-8488499cff-prrb9
10.0.10.227	kube-system	csi-oci-node-qkc9l
10.0.10.100	kube-system	csi-oci-node-xhlhr
10.27.0.2	kube-system	kube-dns-autoscaler-76b7f5ddc4-srnls
10.0.10.100	kube-system	kube-flannel-ds-fq8cq
10.0.10.227	kube-system	kube-flannel-ds-q4h7w
10.0.10.227	kube-system	kube-proxy-sm7xr
10.0.10.100	kube-system	kube-proxy-v6mpc
10.27.0.3	kube-system	kubernetes-dashboard-cd6c8c5b7-r8jmt
10.0.10.227	kube-system	proxymux-client-jr6z4
10.0.10.100	kube-system	proxymux-client-wlpjq
root@hello-python-5dc999f598-qkzj6:/#

Thank you for the support. There was some configuration on my previous cluster then.
Please close this issue.
/close
Regards,
Piotr

[k8s-ci-robot]

@Michalski-Piotr: Closing this issue.

Details

In response to this:

Hello @tomplus,
thank you. You are right, my bad. From pod, in the new cluster all is working fine:

[opc@k8s-registry kubernetes]$ k get pods
NAME                            READY   STATUS    RESTARTS      AGE
busybox                         1/1     Running   3 (18m ago)   3h18m
hello-python-5dc999f598-qkzj6   1/1     Running   0             3m57s
hello-python-5dc999f598-xwdl7   1/1     Running   0             3m56s
[opc@k8s-registry kubernetes]$

$ k exec -it hello-python-5dc999f598-qkzj6 -- /bin/bash -l

cat test.py
from kubernetes import client, config


def main():
   config.load_incluster_config()

   v1 = client.CoreV1Api()
   print("Listing pods with their IPs:")
   ret = v1.list_pod_for_all_namespaces(watch=False)
   for i in ret.items:
       print("%s\t%s\t%s" %
             (i.status.pod_ip, i.metadata.namespace, i.metadata.name))

if __name__ == '__main__':
   main()

root@hello-python-5dc999f598-qkzj6:/# python3.7 test.py
Listing pods with their IPs:
10.27.0.130	default	busybox
10.27.0.139	default	hello-python-5dc999f598-qkzj6
10.27.0.13	default	hello-python-5dc999f598-xwdl7
10.27.0.4	ingress-nginx	ingress-nginx-admission-create-2x24d
10.27.0.131	ingress-nginx	ingress-nginx-admission-patch-5xcbk
10.27.0.5	ingress-nginx	ingress-nginx-controller-8574b6d7c9-hcp6g
None	kube-system	coredns-8488499cff-76kgd
None	kube-system	coredns-8488499cff-prrb9
10.0.10.227	kube-system	csi-oci-node-qkc9l
10.0.10.100	kube-system	csi-oci-node-xhlhr
10.27.0.2	kube-system	kube-dns-autoscaler-76b7f5ddc4-srnls
10.0.10.100	kube-system	kube-flannel-ds-fq8cq
10.0.10.227	kube-system	kube-flannel-ds-q4h7w
10.0.10.227	kube-system	kube-proxy-sm7xr
10.0.10.100	kube-system	kube-proxy-v6mpc
10.27.0.3	kube-system	kubernetes-dashboard-cd6c8c5b7-r8jmt
10.0.10.227	kube-system	proxymux-client-jr6z4
10.0.10.100	kube-system	proxymux-client-wlpjq
root@hello-python-5dc999f598-qkzj6:/#

Thank you for the support. There was some configuration on my previous cluster then.
Please close this issue.
/close
Regards,
Piotr

Instructions for interacting with me using PR comments are available here. If you have questions or suggestions related to my behavior, please file an issue against the kubernetes/test-infra repository.