SharedIndexInformer does not work while watching the kubernetes event #2151
Description
Describe the bug
The SharedIndexInformer does not work while watching the kubernetes event.
Client Version
<dependency>
<groupId>io.kubernetes</groupId>
<artifactId>client-java</artifactId>
<version>13.0.0</version>
</dependency>
Kubernetes Version
1.18
Java Version
Java 8
To Reproduce
See below "Additional context"
Expected behavior
The informer should always work and watch the event of kubernetes.
KubeConfig
N/A
Server (please complete the following information):
- OS: [Linux]
- Environment [container]
- Cloud [Azure SG ]
Additional context
The problem is after several days running, I saw there is no log printed in the log file, and check the data of service-pod map which stored in the memory cache is not correct. Seems the watch stop working.
such as below image:
Based on the timestamp in log file , our IT supporter says "kubernetes apiserver connection timeout at that time".
Is there any suggestion on this ?
Below is my code sample:
public void run() {
if ("Y".equals(this.k8sTemplate.getK8sCfgProperties().getEnable())) {
k8sTemplate.prepareApiClient(0, TimeUnit.SECONDS);
CoreV1Api api = new CoreV1Api();
try {
log.info("try to init the endpoints......");
k8sTemplate.init(api); // initial
log.info("try to watch the endpoints......");
Thread.sleep(500);
RetrieveKubernetesPodsJob.kubInitFlag = true;
Executors.newSingleThreadExecutor().execute(() -> {
k8sTemplate.watch(api); // then create a single thread to trigger the watching
});
}
catch (Exception e) {
log.error("Exp : " + e.toString());
}
}
else {
log.info("The k8s.config.enable is not Y, don't start the k8s init and watch job.");
}
}
public class KubTemplate {
private static final Log log = LogFactory.getLog(KubTemplate.class);
public static SharedInformerFactory sharedInformerFactory = null;
......
public ApiClient prepareApiClient(int timeout, TimeUnit timeUnit) {
ApiClient client = null;
try {
client = ClientBuilder.cluster().build();
Configuration.setDefaultApiClient(client);
}
catch (IOException e) {
log.error("[getApiClient]Exp : " + e.toString());
}
return client;
}
public void init(CoreV1Api api) {
......
}
public void watch(CoreV1Api coreV1Api) {
SharedInformerFactory factory = new SharedInformerFactory();
sharedInformerFactory = factory;
SharedIndexInformer<V1Endpoints> nodeInformer = sharedInformerFactory
.sharedIndexInformerFor((CallGeneratorParams params) -> {
return coreV1Api.listEndpointsForAllNamespacesCall(null, null, null,
null, null, null, params.resourceVersion, null,
params.timeoutSeconds, params.watch, null);
}, V1Endpoints.class, V1EndpointsList.class);
nodeInformer.addEventHandler(new ResourceEventHandler<V1Endpoints>() {
@Override
public void onAdd(V1Endpoints ep) {
String namespace = ep.getMetadata().getNamespace();
String name = ep.getMetadata().getName();
if (StringUtils.hasText(name) && StringUtils.hasText(namespace)
&& !"default".equals(namespace)
&& !"kube-system".equals(namespace)) {
String serviceName = name.toLowerCase().trim();
log.info("[watch] add V1Endpoints " + serviceName + "." + namespace);
// do something
}
}
@Override
public void onUpdate(V1Endpoints oldEp, V1Endpoints newEp) {
String namespace = newEp.getMetadata().getNamespace();
String name = newEp.getMetadata().getName();
if (StringUtils.hasText(name) && StringUtils.hasText(namespace)
&& !"default".equals(namespace)
&& !"kube-system".equals(namespace)) {
log.info("[watch] update V1Endpoints old: "
+ oldEp.getMetadata().getName() + ", new: " + name + "."
+ namespace);
String serviceName = name.toLowerCase().trim();
// do something
}
}
@Override
public void onDelete(V1Endpoints ep, boolean deletedFinalStateUnknown) {
String namespace = ep.getMetadata().getNamespace();
String name = ep.getMetadata().getName();
if (StringUtils.hasText(name) && StringUtils.hasText(namespace)
&& !"default".equals(namespace)
&& !"kube-system".equals(namespace)) {
log.info("[watch] delete V1Endpoints " + name + "." + namespace);
String serviceName = name.toLowerCase().trim();
// do something
}
}
});
sharedInformerFactory.startAllRegisteredInformers();
}
}