Broker pod CrashLoopBackOff : org.apache.pulsar.functions.worker.WorkerService - Error Starting up in worker

[bleacha870721]

Describe the bug

17:43:02.943 [AsyncHttpClient-53-1] WARN  org.apache.pulsar.client.admin.internal.BaseResource - [http://10.233.82.131:8080/admin/v2/persistent/public/functions/assignments] Failed to perform http put request: javax.ws.rs.InternalServerErrorException: HTTP 500 Internal Server Error
17:43:02.952 [main] ERROR org.apache.pulsar.functions.worker.WorkerService - Error Starting up in worker
org.apache.pulsar.client.admin.PulsarAdminException$ServerSideErrorException: HTTP 500 Internal Server Error
	at org.apache.pulsar.client.admin.internal.BaseResource.getApiException(BaseResource.java:204) ~[org.apache.pulsar-pulsar-client-admin-original-2.5.2.jar:2.5.2]
	at org.apache.pulsar.client.admin.internal.BaseResource$1.failed(BaseResource.java:130) ~[org.apache.pulsar-pulsar-client-admin-original-2.5.2.jar:2.5.2]
	at org.glassfish.jersey.client.JerseyInvocation$4.failed(JerseyInvocation.java:1030) ~[org.glassfish.jersey.core-jersey-client-2.27.jar:?]
	at org.glassfish.jersey.client.JerseyInvocation$4.completed(JerseyInvocation.java:1017) ~[org.glassfish.jersey.core-jersey-client-2.27.jar:?]
	at org.glassfish.jersey.client.ClientRuntime.processResponse(ClientRuntime.java:227) ~[org.glassfish.jersey.core-jersey-client-2.27.jar:?]
	at org.glassfish.jersey.client.ClientRuntime.access$200(ClientRuntime.java:85) ~[org.glassfish.jersey.core-jersey-client-2.27.jar:?]
	at org.glassfish.jersey.client.ClientRuntime$2.lambda$response$0(ClientRuntime.java:178) ~[org.glassfish.jersey.core-jersey-client-2.27.jar:?]
	at org.glassfish.jersey.internal.Errors$1.call(Errors.java:272) ~[org.glassfish.jersey.core-jersey-common-2.27.jar:?]
	at org.glassfish.jersey.internal.Errors$1.call(Errors.java:268) ~[org.glassfish.jersey.core-jersey-common-2.27.jar:?]
	at org.glassfish.jersey.internal.Errors.process(Errors.java:316) ~[org.glassfish.jersey.core-jersey-common-2.27.jar:?]
	at org.glassfish.jersey.internal.Errors.process(Errors.java:298) ~[org.glassfish.jersey.core-jersey-common-2.27.jar:?]
	at org.glassfish.jersey.internal.Errors.process(Errors.java:268) ~[org.glassfish.jersey.core-jersey-common-2.27.jar:?]
	at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:312) ~[org.glassfish.jersey.core-jersey-common-2.27.jar:?]
	at org.glassfish.jersey.client.ClientRuntime$2.response(ClientRuntime.java:178) ~[org.glassfish.jersey.core-jersey-client-2.27.jar:?]
	at org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$3.onCompleted(AsyncHttpConnector.java:270) ~[org.apache.pulsar-pulsar-client-admin-original-2.5.2.jar:2.5.2]
	at org.apache.pulsar.client.admin.internal.http.AsyncHttpConnector$3.onCompleted(AsyncHttpConnector.java:261) ~[org.apache.pulsar-pulsar-client-admin-original-2.5.2.jar:2.5.2]
	at org.asynchttpclient.AsyncCompletionHandler.onCompleted(AsyncCompletionHandler.java:66) ~[org.asynchttpclient-async-http-client-2.7.0.jar:?]
	at org.asynchttpclient.netty.NettyResponseFuture.loadContent(NettyResponseFuture.java:222) ~[org.asynchttpclient-async-http-client-2.7.0.jar:?]
	at org.asynchttpclient.netty.NettyResponseFuture.done(NettyResponseFuture.java:257) ~[org.asynchttpclient-async-http-client-2.7.0.jar:?]
	at org.asynchttpclient.netty.handler.AsyncHttpClientHandler.finishUpdate(AsyncHttpClientHandler.java:241) ~[org.asynchttpclient-async-http-client-2.7.0.jar:?]
	at org.asynchttpclient.netty.handler.HttpHandler.handleChunk(HttpHandler.java:113) ~[org.asynchttpclient-async-http-client-2.7.0.jar:?]
	at org.asynchttpclient.netty.handler.HttpHandler.handleRead(HttpHandler.java:142) ~[org.asynchttpclient-async-http-client-2.7.0.jar:?]
	at org.asynchttpclient.netty.handler.AsyncHttpClientHandler.channelRead(AsyncHttpClientHandler.java:78) ~[org.asynchttpclient-async-http-client-2.7.0.jar:?]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) ~[io.netty-netty-codec-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:436) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:321) ~[io.netty-netty-codec-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:295) ~[io.netty-netty-codec-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:251) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:650) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:576) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[io.netty-netty-transport-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) ~[io.netty-netty-common-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[io.netty-netty-common-4.1.48.Final.jar:4.1.48.Final]
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[io.netty-netty-common-4.1.48.Final.jar:4.1.48.Final]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_252]
Caused by: javax.ws.rs.InternalServerErrorException: HTTP 500 Internal Server Error
	at org.glassfish.jersey.client.JerseyInvocation.convertToException(JerseyInvocation.java:1098) ~[org.glassfish.jersey.core-jersey-client-2.27.jar:?]
	at org.glassfish.jersey.client.JerseyInvocation.access$700(JerseyInvocation.java:99) ~[org.glassfish.jersey.core-jersey-client-2.27.jar:?]
	... 47 more

Screenshots

Additional context
Pulsar version: 2.5.2

[bleacha870721]

log.txt

[jiazhai]

@bleacha870721 Are there any different configs in conf/broker.conf, like enabled auth?

[lixiajie]

Yes, we enabled jwt auth.

@bleacha870721 Are there any different configs in conf/broker.conf, like enabled auth?

Yes, we enabled jwt auth.

[lixiajie]

@bleacha870721 Are there any different configs in conf/broker.conf, like enabled auth?

[TC-robV]

ever figure this out? running into the same issues. no auth enabled though.

[tisonkun]

Closed as stale. Please create a new issue if it's still relevant to the maintained versions.