XrdCl::AssignLBHandler::HandleResponseWithHosts #758

[gumond]

We still observe an issue similar to #758 with eos-xrootd-5.3.1-1.el7.cern (on st-096-gg500zp1.cern.ch).

core_backtrace:
:{   "signal": 11
:,   "executable": "/opt/eos/xrootd/bin/xrootd"
:,   "stacktrace":
:      [ {   "crash_thread": true
:        ,   "frames":
:              [ {   "address": 140609874105171
:                ,   "build_id": "09cfb171310110bc7ea9f4476c9fa044d85baff4"
:                ,   "build_id_offset": 381779
:                ,   "function_name": "__dynamic_cast"
:                ,   "file_name": "/lib64/libstdc++.so.6"
:                }
:              , {   "address": 140609703847886
:                ,   "build_id": "9b900d13ef9e7e68d1dcd7d0f2c29d653605bb8c"
:                ,   "build_id_offset": 1152974
:                ,   "function_name": "XrdCl::AssignLastURLHandler::HandleResponseWithHosts(XrdCl::XRootDStatus*, XrdCl::AnyObject*, std::vector<XrdCl::HostInfo, std::allocator<XrdCl::HostInfo> >*)"
:                ,   "file_name": "/opt/eos/xrootd/lib64/libXrdCl.so.3"
:                }
:              , {   "address": 140609703847298
:                ,   "build_id": "9b900d13ef9e7e68d1dcd7d0f2c29d653605bb8c"
:                ,   "build_id_offset": 1152386
:                ,   "function_name": "XrdCl::AssignLBHandler::HandleResponseWithHosts(XrdCl::XRootDStatus*, XrdCl::AnyObject*, std::vector<XrdCl::HostInfo, std::allocator<XrdCl::HostInfo> >*)"
:                ,   "file_name": "/opt/eos/xrootd/lib64/libXrdCl.so.3"
:                }
:              , {   "address": 140609703911027
:                ,   "build_id": "9b900d13ef9e7e68d1dcd7d0f2c29d653605bb8c"
:                ,   "build_id_offset": 1216115
:                ,   "function_name": "XrdCl::XRootDMsgHandler::HandleResponse()"
:                ,   "file_name": "/opt/eos/xrootd/lib64/libXrdCl.so.3"
:                }
:              , {   "address": 140609703925264
:                ,   "build_id": "9b900d13ef9e7e68d1dcd7d0f2c29d653605bb8c"
:                ,   "build_id_offset": 1230352
:                ,   "function_name": "XrdCl::XRootDMsgHandler::Process(XrdCl::Message*)"
:                ,   "file_name": "/opt/eos/xrootd/lib64/libXrdCl.so.3"
:                }
:              , {   "address": 140609703753710
:                ,   "build_id": "9b900d13ef9e7e68d1dcd7d0f2c29d653605bb8c"
:                ,   "build_id_offset": 1058798
:                ,   "function_name": "XrdCl::Stream::HandleIncMsgJob::Run(void*)"
:                ,   "file_name": "/opt/eos/xrootd/lib64/libXrdCl.so.3"
:                }
:              , {   "address": 140609704302381
:                ,   "build_id": "9b900d13ef9e7e68d1dcd7d0f2c29d653605bb8c"
:                ,   "build_id_offset": 1607469
:                ,   "function_name": "XrdCl::JobManager::RunJobs()"
:                ,   "file_name": "/opt/eos/xrootd/lib64/libXrdCl.so.3"
:                }
:              , {   "address": 140609704302505
:                ,   "build_id": "9b900d13ef9e7e68d1dcd7d0f2c29d653605bb8c"
:                ,   "build_id_offset": 1607593
:                ,   "function_name": "RunRunnerThread"
:                ,   "file_name": "/opt/eos/xrootd/lib64/libXrdCl.so.3"
:                }
:              , {   "address": 140609879064229
:                ,   "build_id": "e10cc8f2b932fc3daeda22f8dac5ebb969524e5b"
:                ,   "build_id_offset": 32421
:                ,   "function_name": "start_thread"
:                ,   "file_name": "/lib64/libpthread.so.0"
:                }
:              , {   "address": 140609865435917
:                ,   "build_id": "8da9707d518e88179b1b2903cd846c82102d84c1"
:                ,   "build_id_offset": 1043213
:                ,   "function_name": "__clone"
:                ,   "file_name": "/lib64/libc.so.6"
:                } ]
:        }

[gumond]

(gdb) bt
#0  0x00007f535ed7e353 in __dynamic_cast () from /lib64/libstdc++.so.6
#1  0x00007f5354b327ce in XrdCl::AssignLastURLHandler::HandleResponseWithHosts (this=0x7f533c678ea0, status=0x7f51c5408e40, response=0x7f50a5f75320, hostList=0x7f508892cc80)
    at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClFileSystem.cc:1054
#2  0x00007f5354b32582 in XrdCl::AssignLBHandler::HandleResponseWithHosts (this=0x7f533c678ee0, status=0x7f51c5408e40, response=0x7f50a5f75320, hostList=0x7f508892cc80)
    at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClFileSystem.cc:1032
#3  0x00007f5354b41e73 in XrdCl::XRootDMsgHandler::HandleResponse (this=this@entry=0x7f533c4e8d00) at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClXRootDMsgHandler.cc:1603
#4  0x00007f5354b45610 in XrdCl::XRootDMsgHandler::Process (this=0x7f533c4e8d00, msg=<optimized out>) at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClXRootDMsgHandler.cc:563
#5  0x00007f5354b1b7ee in XrdCl::Stream::HandleIncMsgJob::Run (this=0x7f51c5408e90, arg=<optimized out>) at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClStream.hh:323
#6  0x00007f5354ba172d in XrdCl::JobManager::RunJobs (this=0x7f535bc60c60) at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClJobManager.cc:153
#7  0x00007f5354ba17a9 in RunRunnerThread (arg=<optimized out>) at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClJobManager.cc:34
#8  0x00007f535f238ea5 in start_thread () from /lib64/libpthread.so.0
#9  0x00007f535e5399fd in ioperm () from /lib64/libc.so.6
#10 0x0000000000000000 in ?? ()

few frames:

(gdb) f 1
#1  0x00007f5354b327ce in XrdCl::AssignLastURLHandler::HandleResponseWithHosts (this=0x7f533c678ea0, status=0x7f51c5408e40, response=0x7f50a5f75320, hostList=0x7f508892cc80)
    at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClFileSystem.cc:1054
1054	    SyncResponseHandler *syncHandler = dynamic_cast<SyncResponseHandler*>( pUserHandler );
(gdb) f 2
#2  0x00007f5354b32582 in XrdCl::AssignLBHandler::HandleResponseWithHosts (this=0x7f533c678ee0, status=0x7f51c5408e40, response=0x7f50a5f75320, hostList=0x7f508892cc80)
    at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClFileSystem.cc:1032
1032	      pUserHandler->HandleResponseWithHosts( status, response, hostList );
(gdb) f 3
#3  0x00007f5354b41e73 in XrdCl::XRootDMsgHandler::HandleResponse (this=this@entry=0x7f533c4e8d00) at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClXRootDMsgHandler.cc:1603
1603	    pResponseHandler->HandleResponseWithHosts( status, response, hosts );
(gdb) f 4
#4  0x00007f5354b45610 in XrdCl::XRootDMsgHandler::Process (this=0x7f533c4e8d00, msg=<optimized out>) at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClXRootDMsgHandler.cc:563
563	        HandleResponse();
(gdb) f 5
#5  0x00007f5354b1b7ee in XrdCl::Stream::HandleIncMsgJob::Run (this=0x7f51c5408e90, arg=<optimized out>) at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClStream.hh:323
323	            pHandler->Process( msg );
(gdb) f 6
#6  0x00007f5354ba172d in XrdCl::JobManager::RunJobs (this=0x7f535bc60c60) at /usr/src/debug/xrootd-5.3.1/src/XrdCl/XrdClJobManager.cc:153
153	      h.job->Run( h.arg );

[abh3]

Wasn't this fixed in 5.4?

[simonmichal]

It was observed in 5.3.0 again so it has to be investigated.

[simonmichal]

@simonmichal : this has been fixed.