[Flaky-test] LLCRealtimeClusterIntegrationTest.testReset #11766
Description
Don't see this exception before, so this might be related to the Helix 1.3.1 upgrade #11754. Need further investigation.
Error: org.apache.pinot.integration.tests.LLCRealtimeClusterIntegrationTest.testReset -- Time elapsed: 0.011 s <<< FAILURE!
java.io.IOException: org.apache.pinot.common.exception.HttpErrorStatusException: Got error status code: 500 (Internal Server Error) with reason: "Failed to reset segments in table: mytable_REALTIME. Can't reset state for mytable_REALTIME.[mytable__0__0__20231009T1914Z, mytable__0__15__20231009T1915Z, mytable__1__27__20231009T1918Z, mytable__0__7__20231009T1915Z, mytable__0__12__20231009T1915Z, mytable__0__3__20231009T1914Z, mytable_16405_16435_2 %, mytable__0__22__20231009T1915Z, mytable__1__22__20231009T1915Z, mytable__1__14__20231009T1915Z, mytable__1__11__20231009T1915Z, mytable_16071_16101_3 %, mytable_16130_16159_5 %, mytable__1__26__20231009T1917Z, mytable__0__24__20231009T1917Z, mytable__1__9__20231009T1915Z, mytable__0__18__20231009T1915Z, mytable__1__2__20231009T1914Z, mytable__0__23__20231009T1915Z, mytable_16343_16373_0 %, mytable__0__4__20231009T1914Z, mytable__1__24__20231009T1917Z, mytable__0__11__20231009T1915Z, mytable__0__1__20231009T1914Z, mytable__1__17__20231009T1915Z, mytable__1__6__20231009T1915Z, mytable__1__4__20231009T1914Z, mytable__1__1__20231009T1914Z, mytable_16160_16189_6 %, mytable__0__17__20231009T1915Z, mytable_16313_16342_11 %, mytable_16374_16404_1 %, mytable__0__14__20231009T1915Z, mytable_16102_16129_4 %, mytable__0__20__20231009T1915Z, mytable__1__16__20231009T1915Z, mytable__1__7__20231009T1915Z, mytable__1__13__20231009T1915Z, mytable__1__20__20231009T1915Z, mytable__1__3__20231009T1914Z, mytable__1__0__20231009T1914Z, mytable__1__19__20231009T1915Z, mytable__1__25__20231009T1917Z, mytable_16282_16312_10 %, mytable__0__10__20231009T1915Z, mytable_16190_16220_7 %, mytable__0__9__20231009T1915Z, mytable__0__2__20231009T1914Z, mytable__0__21__20231009T1915Z, mytable__0__25__20231009T1917Z, mytable__0__6__20231009T1915Z, mytable__1__10__20231009T1915Z, mytable__1__23__20231009T1915Z, mytable_16251_16281_9 %, mytable_16221_16250_8 %, mytable__0__16__20231009T1915Z, mytable__0__19__20231009T1915Z, mytable__0__13__20231009T1915Z, mytable__1__5__20231009T1915Z, mytable__0__8__20231009T1915Z, mytable__1__21__20231009T1915Z, mytable__0__26__20231009T1917Z, mytable__1__12__20231009T1915Z, mytable__0__5__20231009T1915Z, mytable__1__8__20231009T1915Z, mytable__1__15__20231009T1915Z, mytable__1__18__20231009T1915Z] on Server_localhost_8098, because a pending message ZnRecord=b1d7d8ce-3d91-4151-8206-94e2a4458fe6, {CREATE_TIMESTAMP=1696879088598, ClusterEventName=IdealStateChange, EXE_SESSION_ID=100001187a50015, FROM_STATE=CONSUMING, MSG_ID=b1d7d8ce-3d91-4151-8206-94e2a4458fe6, MSG_STATE=read, MSG_TYPE=STATE_TRANSITION, PARTITION_NAME=mytable__0__26__20231009T1917Z, READ_TIMESTAMP=1696879088604, RESOURCE_NAME=mytable_REALTIME, RESOURCE_TAG=mytable_REALTIME, RETRY_COUNT=3, SRC_NAME=localhost_18998, SRC_SESSION_ID=100001187a50005, STATE_MODEL_DEF=SegmentOnlineOfflineStateModel, STATE_MODEL_FACTORY_NAME=DEFAULT, TGT_NAME=Server_localhost_8098, TGT_SESSION_ID=100001187a50015, TO_STATE=ONLINE}{}{}, Stat=Stat {_version=0, _creationTime=0, _modifiedTime=0, _ephemeralOwner=0} exists for resource mytable_REALTIME" while sending request: http://localhost:18998/segments/mytable_REALTIME/reset to controller: fv-az1032-428.a2cmwlnf3g0ujp1vqeksrtzoac.cx.internal.cloudapp.net, version: 1.1.0-SNAPSHOT-a78eb37b138fddddeaa7c168b93d2f384131d797
at org.apache.pinot.controller.helix.ControllerRequestClient.resetTable(ControllerRequestClient.java:164)
at org.apache.pinot.integration.tests.BaseClusterIntegrationTest.resetTable(BaseClusterIntegrationTest.java:721)
at org.apache.pinot.integration.tests.BaseClusterIntegrationTestSet.testReset(BaseClusterIntegrationTestSet.java:613)
at org.apache.pinot.integration.tests.LLCRealtimeClusterIntegrationTest.testReset(LLCRealtimeClusterIntegrationTest.java:318)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at org.testng.internal.invokers.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:139)
at org.testng.internal.invokers.TestInvoker.invokeMethod(TestInvoker.java:664)
at org.testng.internal.invokers.TestInvoker.invokeTestMethod(TestInvoker.java:227)
at org.testng.internal.invokers.MethodRunner.runInSequence(MethodRunner.java:50)
at org.testng.internal.invokers.TestInvoker$MethodInvocationAgent.invoke(TestInvoker.java:957)
at org.testng.internal.invokers.TestInvoker.invokeTestMethods(TestInvoker.java:200)
at org.testng.internal.invokers.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:148)
at org.testng.internal.invokers.TestMethodWorker.run(TestMethodWorker.java:128)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1511)
at org.testng.TestRunner.privateRun(TestRunner.java:848)
at org.testng.TestRunner.run(TestRunner.java:621)
at org.testng.SuiteRunner.runTest(SuiteRunner.java:443)
at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:437)
at org.testng.SuiteRunner.privateRun(SuiteRunner.java:397)
at org.testng.SuiteRunner.run(SuiteRunner.java:336)
at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:52)
at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:95)
at org.testng.TestNG.runSuitesSequentially(TestNG.java:1280)
at org.testng.TestNG.runSuitesLocally(TestNG.java:1200)
at org.testng.TestNG.runSuites(TestNG.java:1114)
at org.testng.TestNG.run(TestNG.java:1082)
at org.apache.maven.surefire.testng.TestNGExecutor.run(TestNGExecutor.java:155)
at org.apache.maven.surefire.testng.TestNGDirectoryTestSuite.executeSingleClass(TestNGDirectoryTestSuite.java:102)
at org.apache.maven.surefire.testng.TestNGDirectoryTestSuite.execute(TestNGDirectoryTestSuite.java:91)
at org.apache.maven.surefire.testng.TestNGProvider.invoke(TestNGProvider.java:137)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385)
at org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
at org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495)
Caused by: org.apache.pinot.common.exception.HttpErrorStatusException: Got error status code: 500 (Internal Server Error) with reason: "Failed to reset segments in table: mytable_REALTIME. Can't reset state for mytable_REALTIME.[mytable__0__0__20231009T1914Z, mytable__0__15__20231009T1915Z, mytable__1__27__20231009T1918Z, mytable__0__7__20231009T1915Z, mytable__0__12__20231009T1915Z, mytable__0__3__20231009T1914Z, mytable_16405_16435_2 %, mytable__0__22__20231009T1915Z, mytable__1__22__20231009T1915Z, mytable__1__14__20231009T1915Z, mytable__1__11__20231009T1915Z, mytable_16071_16101_3 %, mytable_16130_16159_5 %, mytable__1__26__20231009T1917Z, mytable__0__24__20231009T1917Z, mytable__1__9__20231009T1915Z, mytable__0__18__20231009T1915Z, mytable__1__2__20231009T1914Z, mytable__0__23__20231009T1915Z, mytable_16343_16373_0 %, mytable__0__4__20231009T1914Z, mytable__1__24__20231009T1917Z, mytable__0__11__20231009T1915Z, mytable__0__1__20231009T1914Z, mytable__1__17__20231009T1915Z, mytable__1__6__20231009T1915Z, mytable__1__4__20231009T1914Z, mytable__1__1__20231009T1914Z, mytable_16160_16189_6 %, mytable__0__17__20231009T1915Z, mytable_16313_16342_11 %, mytable_16374_16404_1 %, mytable__0__14__20231009T1915Z, mytable_16102_16129_4 %, mytable__0__20__20231009T1915Z, mytable__1__16__20231009T1915Z, mytable__1__7__20231009T1915Z, mytable__1__13__20231009T1915Z, mytable__1__20__20231009T1915Z, mytable__1__3__20231009T1914Z, mytable__1__0__20231009T1914Z, mytable__1__19__20231009T1915Z, mytable__1__25__20231009T1917Z, mytable_16282_16312_10 %, mytable__0__10__20231009T1915Z, mytable_16190_16220_7 %, mytable__0__9__20231009T1915Z, mytable__0__2__20231009T1914Z, mytable__0__21__20231009T1915Z, mytable__0__25__20231009T1917Z, mytable__0__6__20231009T1915Z, mytable__1__10__20231009T1915Z, mytable__1__23__20231009T1915Z, mytable_16251_16281_9 %, mytable_16221_16250_8 %, mytable__0__16__20231009T1915Z, mytable__0__19__20231009T1915Z, mytable__0__13__20231009T1915Z, mytable__1__5__20231009T1915Z, mytable__0__8__20231009T1915Z, mytable__1__21__20231009T1915Z, mytable__0__26__20231009T1917Z, mytable__1__12__20231009T1915Z, mytable__0__5__20231009T1915Z, mytable__1__8__20231009T1915Z, mytable__1__15__20231009T1915Z, mytable__1__18__20231009T1915Z] on Server_localhost_8098, because a pending message ZnRecord=b1d7d8ce-3d91-4151-8206-94e2a4458fe6, {CREATE_TIMESTAMP=1696879088598, ClusterEventName=IdealStateChange, EXE_SESSION_ID=100001187a50015, FROM_STATE=CONSUMING, MSG_ID=b1d7d8ce-3d91-4151-8206-94e2a4458fe6, MSG_STATE=read, MSG_TYPE=STATE_TRANSITION, PARTITION_NAME=mytable__0__26__20231009T1917Z, READ_TIMESTAMP=1696879088604, RESOURCE_NAME=mytable_REALTIME, RESOURCE_TAG=mytable_REALTIME, RETRY_COUNT=3, SRC_NAME=localhost_18998, SRC_SESSION_ID=100001187a50005, STATE_MODEL_DEF=SegmentOnlineOfflineStateModel, STATE_MODEL_FACTORY_NAME=DEFAULT, TGT_NAME=Server_localhost_8098, TGT_SESSION_ID=100001187a50015, TO_STATE=ONLINE}{}{}, Stat=Stat {_version=0, _creationTime=0, _modifiedTime=0, _ephemeralOwner=0} exists for resource mytable_REALTIME" while sending request: http://localhost:18998/segments/mytable_REALTIME/reset to controller: fv-az1032-428.a2cmwlnf3g0ujp1vqeksrtzoac.cx.internal.cloudapp.net, version: 1.1.0-SNAPSHOT-a78eb37b138fddddeaa7c168b93d2f384131d797
at org.apache.pinot.common.utils.http.HttpClient.wrapAndThrowHttpException(HttpClient.java:448)
at org.apache.pinot.controller.helix.ControllerRequestClient.resetTable(ControllerRequestClient.java:161)
... 36 more