exception on broker regarding Kafka Indexing Service #6117
Description
seeing this in the logs:
2018-08-06 20:03:02,521 WARN o.j.n.c.SimpleChannelUpstreamHandler [HttpClient-Netty-Boss-0] EXCEPTION, please implement org.jboss.netty.handler.codec.http.HttpContentDecompressor.exceptionCaught() for proper handling.
java.net.ConnectException: Connection refused: /10.240.0.4:8102
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_181]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) ~[?:1.8.0_181]
at org.jboss.netty.channel.socket.nio.NioClientBoss.connect(NioClientBoss.java:152) ~[netty-3.10.6.Final.jar:?]
at org.jboss.netty.channel.socket.nio.NioClientBoss.processSelectedKeys(NioClientBoss.java:105) [netty-3.10.6.Final.jar:?]
at org.jboss.netty.channel.socket.nio.NioClientBoss.process(NioClientBoss.java:79) [netty-3.10.6.Final.jar:?]
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337) [netty-3.10.6.Final.jar:?]
at org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42) [netty-3.10.6.Final.jar:?]
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) [netty-3.10.6.Final.jar:?]
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) [netty-3.10.6.Final.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_181]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_181]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]
And:
2018-08-06 20:03:02,526 ERROR i.d.s.QueryResource [qtp1680479914-145[groupBy_[infraserver-minute]_f88a0605-b2dc-4cba-84c7-a61f1d2c5e29]] Exception handling request: {class=io.druid.server.QueryResource, exceptionType=class io.druid.java.util.common.RE, exceptionMessage=Failure getting results for query[f88a0605-b2dc-4cba-84c7-a61f1d2c5e29] url[http://10.240.0.4:8102/druid/v2/] because of [org.jboss.netty.channel.ChannelException: Faulty channel in resource pool], exception=io.druid.java.util.common.RE: Failure getting results for query[f88a0605-b2dc-4cba-84c7-a61f1d2c5e29] url[http://10.240.0.4:8102/druid/v2/] because of [org.jboss.netty.channel.ChannelException: Faulty channel in resource pool], query=GroupByQuery{dataSource='infraserver-minute', querySegmentSpec=LegacySegmentSpec{intervals=[2018-08-06T20:00:00.000Z/2018-08-06T20:01:00.000Z]}, virtualColumns=[], limitSpec=NoopLimitSpec, dimFilter=(accountid = 57232dcbe4b05a67d2b869e0 && (metric = system.disk.totalbytes || metric = system.disk.usedbytes) && (hostid = 59a986abe4b044c887b017e3)), granularity={type=period, period=PT2880000M, timeZone=UTC, origin=2018-08-06T20:00:00.000Z}, dimensions=[DefaultDimensionSpec{dimension='hostid', outputName='hostid', outputType='STRING'}, DefaultDimensionSpec{dimension='metric', outputName='metric', outputType='STRING'}, DefaultDimensionSpec{dimension='entity', outputName='entity', outputType='STRING'}], aggregatorSpecs=[DoubleSumAggregatorFactory{fieldName='value', expression='null', name='valuetmp'}], postAggregatorSpecs=[ArithmeticPostAggregator{name='value', fnName='/', fields=[FieldAccessPostAggregator{name='null', fieldName='valuetmp'}, ConstantPostAggregator{name='elemCount', constantValue=1.0}], op=DIV}], havingSpec=null}, peer=10.240.0.12}
io.druid.java.util.common.RE: Failure getting results for query[f88a0605-b2dc-4cba-84c7-a61f1d2c5e29] url[http://10.240.0.4:8102/druid/v2/] because of [org.jboss.netty.channel.ChannelException: Faulty channel in resource pool]
at io.druid.client.DirectDruidClient$JsonParserIterator.init(DirectDruidClient.java:628) ~[druid-server-0.12.1.jar:0.12.1]
at io.druid.client.DirectDruidClient$JsonParserIterator.hasNext(DirectDruidClient.java:560) ~[druid-server-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.BaseSequence.makeYielder(BaseSequence.java:88) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.BaseSequence.toYielder(BaseSequence.java:68) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.MappedSequence.toYielder(MappedSequence.java:49) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.MergeSequence.lambda$toYielder$0(MergeSequence.java:66) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.MappingAccumulator.accumulate(MappingAccumulator.java:40) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.BaseSequence.accumulate(BaseSequence.java:46) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.MappedSequence.accumulate(MappedSequence.java:43) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.MergeSequence.toYielder(MergeSequence.java:63) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.LazySequence.toYielder(LazySequence.java:46) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.query.RetryQueryRunner$1.toYielder(RetryQueryRunner.java:102) ~[druid-processing-0.12.1.jar:0.12.1]
at io.druid.common.guava.CombiningSequence.toYielder(CombiningSequence.java:80) ~[druid-common-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.MappedSequence.toYielder(MappedSequence.java:49) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.WrappingSequence$2.get(WrappingSequence.java:87) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.WrappingSequence$2.get(WrappingSequence.java:83) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.SequenceWrapper.wrap(SequenceWrapper.java:55) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.WrappingSequence.toYielder(WrappingSequence.java:82) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.MappedSequence.toYielder(MappedSequence.java:49) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.WrappingSequence$2.get(WrappingSequence.java:87) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.WrappingSequence$2.get(WrappingSequence.java:83) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.query.CPUTimeMetricQueryRunner$1.wrap(CPUTimeMetricQueryRunner.java:74) ~[druid-processing-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.WrappingSequence.toYielder(WrappingSequence.java:82) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.java.util.common.guava.Yielders.each(Yielders.java:32) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.server.QueryResource.doPost(QueryResource.java:193) [druid-server-0.12.1.jar:0.12.1]
at sun.reflect.GeneratedMethodAccessor69.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_181]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_181]
at com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409) [jersey-server-1.19.3.jar:1.19.3]
at com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409) [jersey-servlet-1.19.3.jar:1.19.3]
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:558) [jersey-servlet-1.19.3.jar:1.19.3]
at com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:733) [jersey-servlet-1.19.3.jar:1.19.3]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) [javax.servlet-api-3.1.0.jar:3.1.0]
at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:286) [guice-servlet-4.1.0.jar:?]
at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:276) [guice-servlet-4.1.0.jar:?]
at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:181) [guice-servlet-4.1.0.jar:?]
at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91) [guice-servlet-4.1.0.jar:?]
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85) [guice-servlet-4.1.0.jar:?]
at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:120) [guice-servlet-4.1.0.jar:?]
at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:135) [guice-servlet-4.1.0.jar:?]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [jetty-servlet-9.3.19.v20170502.jar:9.3.19.v20170502]
at io.druid.server.security.PreResponseAuthorizationCheckFilter.doFilter(PreResponseAuthorizationCheckFilter.java:84) [druid-server-0.12.1.jar:0.12.1]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [jetty-servlet-9.3.19.v20170502.jar:9.3.19.v20170502]
at io.druid.server.security.AllowOptionsResourceFilter.doFilter(AllowOptionsResourceFilter.java:76) [druid-server-0.12.1.jar:0.12.1]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [jetty-servlet-9.3.19.v20170502.jar:9.3.19.v20170502]
at io.druid.server.security.AllowAllAuthenticator$1.doFilter(AllowAllAuthenticator.java:85) [druid-server-0.12.1.jar:0.12.1]
at io.druid.server.security.AuthenticationWrappingFilter.doFilter(AuthenticationWrappingFilter.java:60) [druid-server-0.12.1.jar:0.12.1]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [jetty-servlet-9.3.19.v20170502.jar:9.3.19.v20170502]
at io.druid.server.security.SecuritySanityCheckFilter.doFilter(SecuritySanityCheckFilter.java:86) [druid-server-0.12.1.jar:0.12.1]
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759) [jetty-servlet-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582) [jetty-servlet-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512) [jetty-servlet-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.Server.handle(Server.java:534) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251) [jetty-server-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283) [jetty-io-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108) [jetty-io-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93) [jetty-io-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303) [jetty-util-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) [jetty-util-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) [jetty-util-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) [jetty-util-9.3.19.v20170502.jar:9.3.19.v20170502]
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) [jetty-util-9.3.19.v20170502.jar:9.3.19.v20170502]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_181]
Caused by: java.util.concurrent.ExecutionException: org.jboss.netty.channel.ChannelException: Faulty channel in resource pool
at com.google.common.util.concurrent.Futures$ImmediateFailedFuture.get(Futures.java:186) ~[guava-16.0.1.jar:?]
at io.druid.client.DirectDruidClient$JsonParserIterator.init(DirectDruidClient.java:598) ~[druid-server-0.12.1.jar:0.12.1]
... 81 more
Caused by: org.jboss.netty.channel.ChannelException: Faulty channel in resource pool
at io.druid.java.util.http.client.NettyHttpClient.go(NettyHttpClient.java:147) ~[java-util-0.12.1.jar:0.12.1]
at io.druid.client.DirectDruidClient.run(DirectDruidClient.java:440) ~[druid-server-0.12.1.jar:0.12.1]
at io.druid.client.CachingClusteredClient$SpecificQueryRunnable.getSimpleServerResults(CachingClusteredClient.java:585) ~[druid-server-0.12.1.jar:0.12.1]
at io.druid.client.CachingClusteredClient$SpecificQueryRunnable.lambda$addSequencesFromServer$6(CachingClusteredClient.java:554) ~[druid-server-0.12.1.jar:0.12.1]
at java.util.TreeMap.forEach(TreeMap.java:1005) ~[?:1.8.0_181]
at io.druid.client.CachingClusteredClient$SpecificQueryRunnable.addSequencesFromServer(CachingClusteredClient.java:540) ~[druid-server-0.12.1.jar:0.12.1]
at io.druid.client.CachingClusteredClient$SpecificQueryRunnable.lambda$run$1(CachingClusteredClient.java:282) ~[druid-server-0.12.1.jar:0.12.1]
... 72 more
Caused by: java.net.ConnectException: Connection refused: /10.240.0.4:8102
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_181]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717) ~[?:1.8.0_181]
at org.jboss.netty.channel.socket.nio.NioClientBoss.connect(NioClientBoss.java:152) ~[netty-3.10.6.Final.jar:?]
at org.jboss.netty.channel.socket.nio.NioClientBoss.processSelectedKeys(NioClientBoss.java:105) ~[netty-3.10.6.Final.jar:?]
at org.jboss.netty.channel.socket.nio.NioClientBoss.process(NioClientBoss.java:79) ~[netty-3.10.6.Final.jar:?]
at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337) ~[netty-3.10.6.Final.jar:?]
at org.jboss.netty.channel.socket.nio.NioClientBoss.run(NioClientBoss.java:42) ~[netty-3.10.6.Final.jar:?]
at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) ~[netty-3.10.6.Final.jar:?]
at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) ~[netty-3.10.6.Final.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_181]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_181]
... 1 more
10.240.0.4 is one of the middle manager nodes running kafka indexing service tasks. From time to time we see these errors when using kafka indexing service.
Druid 0.12.1