[Tests] LockManagerImpl.close hangs and causes the test run to timeout eventually #13092
Description
Describe the bug
LockManagerImpl.close sporadically hangs and causes the test run to timeout eventually.
Here's an example thread stack trace:
"main" #1 prio=5 os_prio=0 cpu=48792.59ms elapsed=3273.19s tid=0x00007f2ac0028000 nid=0xc4e waiting on condition [0x00007f2ac7ca7000]
java.lang.Thread.State: WAITING (parking)
at jdk.internal.misc.Unsafe.park(java.base@11.0.13/Native Method)
- parking to wait for <0x00000000ca4007f8> (a java.util.concurrent.CompletableFuture$Signaller)
at java.util.concurrent.locks.LockSupport.park(java.base@11.0.13/LockSupport.java:194)
at java.util.concurrent.CompletableFuture$Signaller.block(java.base@11.0.13/CompletableFuture.java:1796)
at java.util.concurrent.ForkJoinPool.managedBlock(java.base@11.0.13/ForkJoinPool.java:3128)
at java.util.concurrent.CompletableFuture.waitingGet(java.base@11.0.13/CompletableFuture.java:1823)
at java.util.concurrent.CompletableFuture.join(java.base@11.0.13/CompletableFuture.java:2043)
at org.apache.pulsar.metadata.coordination.impl.LockManagerImpl.close(LockManagerImpl.java:163)
at org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl.stop(ModularLoadManagerImpl.java:902)
at org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerWrapper.stop(ModularLoadManagerWrapper.java:107)
at org.apache.pulsar.broker.PulsarService.closeAsync(PulsarService.java:421)
at org.apache.pulsar.broker.PulsarService$MockitoMock$1723093763.closeAsync$accessor$h6Cy2gqF(Unknown Source)
at org.apache.pulsar.broker.PulsarService$MockitoMock$1723093763$auxiliary$jyJcdii6.call(Unknown Source)
at org.mockito.internal.invocation.RealMethod$FromCallable$1.call(RealMethod.java:40)
at org.mockito.internal.invocation.RealMethod$FromBehavior.invoke(RealMethod.java:62)
at org.mockito.internal.invocation.InterceptedInvocation.callRealMethod(InterceptedInvocation.java:141)
at org.mockito.internal.stubbing.answers.CallsRealMethods.answer(CallsRealMethods.java:44)
at org.mockito.Answers.answer(Answers.java:98)
at org.mockito.internal.handler.MockHandlerImpl.handle(MockHandlerImpl.java:106)
at org.mockito.internal.handler.NullResultGuardian.handle(NullResultGuardian.java:29)
at org.mockito.internal.handler.InvocationNotifierHandler.handle(InvocationNotifierHandler.java:33)
at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:82)
at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:56)
at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor$DispatcherDefaultingToRealMethod.interceptSuperCallable(MockMethodInterceptor.java:141)
at org.apache.pulsar.broker.PulsarService$MockitoMock$1723093763.closeAsync(Unknown Source)
at org.apache.pulsar.broker.PulsarService.close(PulsarService.java:352)
at org.apache.pulsar.broker.PulsarService$MockitoMock$1723093763.close$accessor$h6Cy2gqF(Unknown Source)
at org.apache.pulsar.broker.PulsarService$MockitoMock$1723093763$auxiliary$fMYlBnS4.call(Unknown Source)
at org.mockito.internal.invocation.RealMethod$FromCallable$1.call(RealMethod.java:40)
at org.mockito.internal.invocation.RealMethod$FromBehavior.invoke(RealMethod.java:62)
at org.mockito.internal.invocation.InterceptedInvocation.callRealMethod(InterceptedInvocation.java:141)
at org.mockito.internal.stubbing.answers.CallsRealMethods.answer(CallsRealMethods.java:44)
at org.mockito.Answers.answer(Answers.java:98)
at org.mockito.internal.handler.MockHandlerImpl.handle(MockHandlerImpl.java:106)
at org.mockito.internal.handler.NullResultGuardian.handle(NullResultGuardian.java:29)
at org.mockito.internal.handler.InvocationNotifierHandler.handle(InvocationNotifierHandler.java:33)
at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:82)
at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:56)
at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor$DispatcherDefaultingToRealMethod.interceptSuperCallable(MockMethodInterceptor.java:141)
at org.apache.pulsar.broker.PulsarService$MockitoMock$1723093763.close(Unknown Source)
at org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.stopBroker(MockedPulsarServiceBaseTest.java:269)
at org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest.internalCleanup(MockedPulsarServiceBaseTest.java:213)
at org.apache.pulsar.broker.admin.NamespacesTest.cleanup(NamespacesTest.java:196)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@11.0.13/Native Method)
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@11.0.13/NativeMethodAccessorImpl.java:62)
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@11.0.13/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(java.base@11.0.13/Method.java:566)
at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:132)
at org.testng.internal.MethodInvocationHelper.invokeMethodConsideringTimeout(MethodInvocationHelper.java:61)
at org.testng.internal.ConfigInvoker.invokeConfigurationMethod(ConfigInvoker.java:366)
at org.testng.internal.ConfigInvoker.invokeConfigurations(ConfigInvoker.java:320)
at org.testng.internal.TestInvoker.runConfigMethods(TestInvoker.java:701)
at org.testng.internal.TestInvoker.runAfterGroupsConfigurations(TestInvoker.java:677)
at org.testng.internal.TestInvoker.invokeMethod(TestInvoker.java:661)
at org.testng.internal.TestInvoker.invokeTestMethod(TestInvoker.java:174)
at org.testng.internal.MethodRunner.runInSequence(MethodRunner.java:46)
at org.testng.internal.TestInvoker$MethodInvocationAgent.invoke(TestInvoker.java:822)
at org.testng.internal.TestInvoker.invokeTestMethods(TestInvoker.java:147)
at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:146)
at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:128)
at org.testng.TestRunner$$Lambda$183/0x00000001002a2840.accept(Unknown Source)
at java.util.ArrayList.forEach(java.base@11.0.13/ArrayList.java:1541)
at org.testng.TestRunner.privateRun(TestRunner.java:764)
at org.testng.TestRunner.run(TestRunner.java:585)
at org.testng.SuiteRunner.runTest(SuiteRunner.java:384)
at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:378)
at org.testng.SuiteRunner.privateRun(SuiteRunner.java:337)
at org.testng.SuiteRunner.run(SuiteRunner.java:286)
at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:53)
at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:96)
at org.testng.TestNG.runSuitesSequentially(TestNG.java:1218)
at org.testng.TestNG.runSuitesLocally(TestNG.java:1140)
at org.testng.TestNG.runSuites(TestNG.java:1069)
at org.testng.TestNG.run(TestNG.java:1037)
at org.apache.maven.surefire.testng.TestNGExecutor.run(TestNGExecutor.java:135)
at org.apache.maven.surefire.testng.TestNGDirectoryTestSuite.executeSingleClass(TestNGDirectoryTestSuite.java:112)
at org.apache.maven.surefire.testng.TestNGDirectoryTestSuite.executeLazy(TestNGDirectoryTestSuite.java:123)
at org.apache.maven.surefire.testng.TestNGDirectoryTestSuite.execute(TestNGDirectoryTestSuite.java:90)
at org.apache.maven.surefire.testng.TestNGProvider.invoke(TestNGProvider.java:146)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:384)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:345)
at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:126)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:418)