Kafka supervisor with unresolvable bootstrap.servers blocks startup

[gianm]

Kafka supervisors with unresolvable bootstrap.servers will block overlord startup, because the KafkaConsumer can't be created. They should be more robust to the KafkaConsumer failing to start up; perhaps retrying periodically in the background to create the consumer would be a good idea.

The stack trace looks like this:

2018-08-06T14:10:00,850 ERROR [LeaderSelector[/druid/overlord/_OVERLORD]] io.druid.curator.discovery.CuratorDruidLeaderSelector - listener becomeLeader() failed. Unable to become leader: {class=io.druid
.curator.discovery.CuratorDruidLeaderSelector, exceptionType=class java.lang.RuntimeException, exceptionMessage=java.lang.reflect.InvocationTargetException}
java.lang.RuntimeException: java.lang.reflect.InvocationTargetException
        at com.google.common.base.Throwables.propagate(Throwables.java:160) ~[guava-16.0.1.jar:?]
        at io.druid.indexing.overlord.TaskMaster$1.becomeLeader(TaskMaster.java:144) ~[druid-indexing-service-0.12.1-iap8.jar:0.12.1-iap8]
        at io.druid.curator.discovery.CuratorDruidLeaderSelector$1.isLeader(CuratorDruidLeaderSelector.java:98) [druid-server-0.12.1-iap8.jar:0.12.1-iap8]
        at org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:665) [curator-recipes-4.0.0.jar:4.0.0]
        at org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:661) [curator-recipes-4.0.0.jar:4.0.0]
        at org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:93) [curator-framework-4.0.0.jar:4.0.0]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
Caused by: java.lang.reflect.InvocationTargetException
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
        at io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler.start(Lifecycle.java:413) ~[java-util-0.12.1-iap8.jar:0.12.1-iap8]
        at io.druid.java.util.common.lifecycle.Lifecycle.start(Lifecycle.java:311) ~[java-util-0.12.1-iap8.jar:0.12.1-iap8]
        at io.druid.indexing.overlord.TaskMaster$1.becomeLeader(TaskMaster.java:141) ~[druid-indexing-service-0.12.1-iap8.jar:0.12.1-iap8]
        ... 7 more
Caused by: org.apache.kafka.common.KafkaException: Failed to construct kafka consumer
        at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:717) ~[?:?]
        at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:597) ~[?:?]
        at io.druid.indexing.kafka.supervisor.KafkaSupervisor.getKafkaConsumer(KafkaSupervisor.java:904) ~[?:?]
        at io.druid.indexing.kafka.supervisor.KafkaSupervisor.start(KafkaSupervisor.java:354) ~[?:?]
        at io.druid.indexing.overlord.supervisor.SupervisorManager.createAndStartSupervisorInternal(SupervisorManager.java:231) ~[druid-indexing-service-0.12.1-iap8.jar:0.12.1-iap8]
        at io.druid.indexing.overlord.supervisor.SupervisorManager.start(SupervisorManager.java:105) ~[druid-indexing-service-0.12.1-iap8.jar:0.12.1-iap8]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
        at io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler.start(Lifecycle.java:413) ~[java-util-0.12.1-iap8.jar:0.12.1-iap8]
        at io.druid.java.util.common.lifecycle.Lifecycle.start(Lifecycle.java:311) ~[java-util-0.12.1-iap8.jar:0.12.1-iap8]
        at io.druid.indexing.overlord.TaskMaster$1.becomeLeader(TaskMaster.java:141) ~[druid-indexing-service-0.12.1-iap8.jar:0.12.1-iap8]
        ... 7 more
Caused by: org.apache.kafka.common.config.ConfigException: No resolvable bootstrap urls given in bootstrap.servers
        at org.apache.kafka.clients.ClientUtils.parseAndValidateAddresses(ClientUtils.java:60) ~[?:?]
        at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:654) ~[?:?]
        at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:597) ~[?:?]
        at io.druid.indexing.kafka.supervisor.KafkaSupervisor.getKafkaConsumer(KafkaSupervisor.java:904) ~[?:?]
        at io.druid.indexing.kafka.supervisor.KafkaSupervisor.start(KafkaSupervisor.java:354) ~[?:?]
        at io.druid.indexing.overlord.supervisor.SupervisorManager.createAndStartSupervisorInternal(SupervisorManager.java:231) ~[druid-indexing-service-0.12.1-iap8.jar:0.12.1-iap8]
        at io.druid.indexing.overlord.supervisor.SupervisorManager.start(SupervisorManager.java:105) ~[druid-indexing-service-0.12.1-iap8.jar:0.12.1-iap8]
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
        at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
        at io.druid.java.util.common.lifecycle.Lifecycle$AnnotationBasedHandler.start(Lifecycle.java:413) ~[java-util-0.12.1-iap8.jar:0.12.1-iap8]
        at io.druid.java.util.common.lifecycle.Lifecycle.start(Lifecycle.java:311) ~[java-util-0.12.1-iap8.jar:0.12.1-iap8]
        at io.druid.indexing.overlord.TaskMaster$1.becomeLeader(TaskMaster.java:141) ~[druid-indexing-service-0.12.1-iap8.jar:0.12.1-iap8]
        ... 7 more

[b-slim]

fixed by #6383