预测出core，能帮忙看下啥原因不？paddle训练和预测的版本都是v1.3.0

[xiuechen]

#0  tcmalloc::CentralFreeList::FetchFromSpans (this=0x189eac0 <tcmalloc::Static::central_cache_+2304>) at src/central_freelist.cc:262
#1  0x0000000000858311 in tcmalloc::CentralFreeList::FetchFromSpansSafe (this=0x189eac0 <tcmalloc::Static::central_cache_+2304>)
    at src/central_freelist.cc:247
#2  0x00000000008583a1 in tcmalloc::CentralFreeList::RemoveRange (this=0x189eac0 <tcmalloc::Static::central_cache_+2304>, 
    start=0x7f57adce8878, end=0x7f57adce8880, N=6) at src/central_freelist.cc:227
#3  0x000000000085957b in tcmalloc::ThreadCache::FetchFromCentralCache (this=0x1c97000, cl=<optimized out>, byte_size=16)
    at src/static_vars.h:59
#4  0x00000000008529de in tcmalloc::ThreadCache::Allocate (this=0x189eac0 <tcmalloc::Static::central_cache_+2304>, size=0, 
    cl=140014554876024) at src/thread_cache.h:329
#5  0x00000000008529de in CheckedMallocResult (result=<optimized out>)
#6  0x00000000008529de in (anonymous namespace)::do_malloc (size=16)
#7  0x00000000008529de in (anonymous namespace)::cpp_alloc (size=<optimized out>, nothrow=false)
#8  0x00000000008de4da in tc_new (size=25815744, size@entry=16) at src/tcmalloc.cc:1521
#9  0x00007f5954e0aa81 in __gnu_cxx::new_allocator<std::__detail::_Hash_node_base*>::allocate (this=<synthetic pointer>, __n=<optimized out>)
    at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/ext/new_allocator.h:104
#10 std::_Hashtable<std::string, std::pair<std::string const, Xbyak::JmpLabel const>, std::allocator<std::pair<std::string const, Xbyak::JmpLabel const> >, std::__detail::_Select1st, std::equal_to<std::string>, std::hash<std::string>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, false> >::_M_allocate_buckets (
    this=0x7f57adce8990, __n=<optimized out>) at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/hashtable.h:779
#11 std::_Hashtable<std::string, std::pair<std::string const, Xbyak::JmpLabel const>, std::allocator<std::pair<std::string const, Xbyak::JmpLabel const> >, std::__detail::_Select1st, std::equal_to<std::string>, std::hash<std::string>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, false> >::_Hashtable(std::_Hashtable<std::string, std::pair<std::string const, Xbyak::JmpLabel const>, std::allocator<std::pair<std::string const, Xbyak::JmpLabel const> >, std::__detail::_Select1st, std::equal_to<std::string>, std::hash<std::string>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, false> >&&) (this=<optimized out>, 
    __ht=<unknown type in ./fluid_inference/paddle/lib/libpaddle_fluid.so, CU 0x798c45d, DIE 0x79e7754>)
    at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/hashtable.h:945
#12 0x00007f5954e0ab55 in std::unordered_multimap<std::basic_string<char, std::char_traits<char>, std::allocator<char> >, Xbyak::JmpLabel const, std::hash<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::basic_string<char, std::char_traits<---Type <return> to continue, or q <return> to quit---
char>, std::allocator<char> > >, std::allocator<std::pair<std::basic_string<char, std::char_traits<char>, std::allocator<char> > const, Xbyak::JmpLabel const> > >::unordered_multimap (this=0x46f31e40)
    at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/unordered_map.h:852
#13 Xbyak::LabelManager::SlabelState::SlabelState (this=0x46f31e10)
    at /home/work/chenxiue/Paddle/build/third_party/install/xbyak/include/xbyak/xbyak.h:1146
#14 std::_List_node<Xbyak::LabelManager::SlabelState>::_List_node<Xbyak::LabelManager::SlabelState> (this=<optimized out>)
    at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/stl_list.h:114
#15 __gnu_cxx::new_allocator<std::_List_node<Xbyak::LabelManager::SlabelState> >::construct<std::_List_node<Xbyak::LabelManager::SlabelState>, Xbyak::LabelManager::SlabelState> (__p=<optimized out>, this=0x472cc090)
    at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/ext/new_allocator.h:120
#16 std::list<Xbyak::LabelManager::SlabelState, std::allocator<Xbyak::LabelManager::SlabelState> >::_M_create_node<Xbyak::LabelManager::SlabelState> (this=0x472cc090) at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/stl_list.h:505
#17 std::list<Xbyak::LabelManager::SlabelState, std::allocator<Xbyak::LabelManager::SlabelState> >::_M_insert<Xbyak::LabelManager::SlabelState> (__position=..., this=0x472cc090) at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/stl_list.h:1561
#18 std::list<Xbyak::LabelManager::SlabelState, std::allocator<Xbyak::LabelManager::SlabelState> >::push_back(Xbyak::LabelManager::SlabelState&&) (__x=<unknown type in ./fluid_inference/paddle/lib/libpaddle_fluid.so, CU 0x798c45d, DIE 0x79e7959>, this=0x472cc090)
    at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/bits/stl_list.h:1021
#19 Xbyak::LabelManager::reset (this=0x472cc088) at /home/work/chenxiue/Paddle/build/third_party/install/xbyak/include/xbyak/xbyak.h:1245
#20 0x00007f5954e35a5d in Xbyak::LabelManager::LabelManager (this=0x472cc088)
    at /home/work/chenxiue/Paddle/build/third_party/install/xbyak/include/xbyak/xbyak.h:1238
#21 Xbyak::CodeGenerator::CodeGenerator (allocator=0x0, userPtr=0x0, maxSize=<optimized out>, this=0x472cc008)
    at /home/work/chenxiue/Paddle/build/third_party/install/xbyak/include/xbyak/xbyak.h:2427
#22 paddle::operators::jit::gen::JitCode::JitCode (code_ptr=0x0, code_size=<optimized out>, this=0x472cc000)
    at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/gen/jitcode.h:70
#23 paddle::operators::jit::gen::SeqPoolJitCode::SeqPoolJitCode (code_ptr=0x0, code_size=<optimized out>, attr=..., this=0x472cc000)
    at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/gen/seqpool.h:32
#24 paddle::operators::jit::make_unique<paddle::operators::jit::gen::SeqPoolJitCode, paddle::operators::jit::seq_pool_attr_s const&, unsigned long> () at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/registry.h:33
#25 paddle::operators::jit::gen::SeqPoolCreator::CreateJitCode (this=<optimized out>, attr=...)
---Type <return> to continue, or q <return> to quit---
    at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/gen/seqpool.cc:74
#26 0x00007f5954c63114 in paddle::operators::jit::GetJitCode<(paddle::operators::jit::KernelType)21, paddle::operators::jit::SeqPoolTuples<float>, paddle::platform::CPUPlace> (attr=...) at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/helper.h:54
#27 0x00007f5954c632eb in paddle::operators::jit::Get<(paddle::operators::jit::KernelType)21, paddle::operators::jit::SeqPoolTuples<float>, paddle::platform::CPUPlace> (attr=...) at /home/work/chenxiue/Paddle/paddle/fluid/operators/jit/helper.h:98


#28 0x00007f59554d87ca in paddle::operators::math::SequencePoolFunctor<paddle::platform::CPUDeviceContext, float>::operator() (

    this=this@entry=0x7f57adce8f9e, context=..., pooltype="SUM", input=..., output=output@entry=0x7f0da68, is_test=is_test@entry=true, 
    index=index@entry=0x0) at /home/work/chenxiue/Paddle/paddle/fluid/operators/math/sequence_pooling.cc:260
#29 0x00007f5954de2155 in paddle::operators::SequencePoolKernel<paddle::platform::CPUDeviceContext, float>::Compute (

    this=this@entry=0x7f57adce9640, context=...) at /home/work/chenxiue/Paddle/paddle/fluid/operators/sequence_ops/sequence_pool_op.h:61
#30 0x00007f5954de2a93 in paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CPUPlace, false, 0ul, paddle::operators::SequencePoolKernel<paddle::platform::CPUDeviceContext, float> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}::operator()(paddle::framework::ExecutionContext const&) const (ctx=..., __closure=<optimized out>)
    at /home/work/chenxiue/Paddle/paddle/fluid/framework/op_registry.h:107
#31 std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CPUPlace, false, 0ul, paddle::operators::SequencePoolKernel<paddle::platform::CPUDeviceContext, float> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) (__functor=..., __args#0=...) at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/functional:2071
#32 0x00007f5955554c53 in std::function<void (paddle::framework::ExecutionContext const&)>::operator()(paddle::framework::ExecutionContext const&) const (__args#0=..., this=0x1e01e70) at /home/opt/gcc-4.8.2.bpkg-r4/gcc-4.8.2.bpkg-r4/include/c++/4.8.2/functional:2471
#33 paddle::framework::OperatorWithKernel::RunImpl (this=0x7f9cdc0, scope=..., place=...)
    at /home/work/chenxiue/Paddle/paddle/fluid/framework/operator.cc:963
#34 0x00007f595555377b in paddle::framework::OperatorBase::Run (this=0x7f9cdc0, scope=..., place=...)
    at /home/work/chenxiue/Paddle/paddle/fluid/framework/operator.cc:185
#35 0x00007f59549dd04a in paddle::framework::Executor::RunPreparedContext (this=0x77f31d0, ctx=0x7f32120, scope=scope@entry=0x7efbcc0, 
    create_local_scope=create_local_scope@entry=false, create_vars=create_vars@entry=false, keep_kids=keep_kids@entry=false)
    at /home/work/chenxiue/Paddle/paddle/fluid/framework/executor.cc:433
#36 0x00007f595489ca41 in paddle::NativePaddlePredictor::Run (this=0x7d48540, inputs=..., output_data=0x7f57adce9fe0, 
    batch_size=<optimized out>) at /home/work/chenxiue/Paddle/paddle/fluid/inference/api/api_impl.cc:148
---Type <return> to continue, or q <return> to quit---
#37 0x00000000004a0647 in baidu::pmallrec::GbdtModel::predict (this=0x1e85240, user_id=18446744073709550617, user_feature=0x46fdc7b0, 
    recallpidlist=std::vector of length 110, capacity 128 = {...}, pidlist=std::vector of length 0, capacity 0, get_dict=true, 
    log_id="40534463", statusstr="") at baidu/netdisk/dp-pmall-recapi/src/baidu/pmallrec/gbdt_model.cpp:475

[NHZlX]

predictor的创建是在线程外创建的么

[xiuechen]

predictor的创建是在线程外创建的么

是在线程外创建的，补充一下，调整了brpc的线程数是1的时候，压测都没问题，可以正常预测。但是压测的情况下会出core，信息：

terminate called after throwing an instance of 'paddle::platform::EnforceNotMet'
what(): Invoke operator lookup_table error.
Python Callstacks:
File "/home/work/gongxijun/anaconda2/lib/python2.7/site-packages/paddle/fluid/framework.py", line 1317, in append_op
attrs=kwargs.get("attrs", None))
File "/home/work/gongxijun/anaconda2/lib/python2.7/site-packages/paddle/fluid/layer_helper.py", line 56, in append_op
return self.main_program.current_block().append_op(*args, **kwargs)
File "/home/work/gongxijun/anaconda2/lib/python2.7/site-packages/paddle/fluid/layers/nn.py", line 364, in embedding
'padding_idx': padding_idx
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/feature/extra_feature.py", line 118, in get_item_embedding
is_sparse=is_sparse)
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/feature/extra_feature.py", line 303, in get_item_combined_features_lr
is_sparse=True))
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/algorithm/gbdt_lr_trainer.py", line 266, in net_program
usr_combined_features = extra_feature.get_item_combined_features_lr(self.user_feature_col)
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/algorithm/gbdt_lr_trainer.py", line 163, in load
[inference, auc_var, avg_cost, acc] = self.net_program()
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/algorithm/gbdt_lr_trainer.py", line 299, in train
[inference, auc_var, avg_cost, acc] = self.load()
File "/home/work/gongxijun/online/rank/huangdao/huangdao/training/train_factory.py", line 55, in start
_train.train()
File "huangdao/huangdao_main.py", line 42, in run
self.__trainer.start()
File "huangdao/huangdao_main.py", line 47, in
Huangdao().run()

C++ Callstacks:
Enforce failed. Expected ids[i] < row_number, but received ids[i]:4607182418800017408 >= row_number:21.
at [/home/work/chenxiue/Paddle/paddle/fluid/operators/lookup_table_op.h:86]
PaddlePaddle Call Stacks:
0 0x7ffcd6982471p void paddle::platform::EnforceNotMet::Initstd::string(std::string, char const*, int) + 1537
1 0x7ffcd6983c1ap paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) + 90
2 0x7ffcd72a6a48p paddle::operators::LookupTableKernel::Compute(paddle::framework::ExecutionContext const&) const + 2824
3 0x7ffcd72a6ef3p std::Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CPUPlace, false, 0ul, paddle::operators::LookupTableKernel, paddle::operators::LookupTableKernel >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::M_invoke(std::Any_data const&, paddle::framework::ExecutionContext const&) + 35
4 0x7ffcd763ec53p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 659
5 0x7ffcd763d77bp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 267
6 0x7ffcd6ac704ap paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 218
7 0x7ffcd6986a41p paddle::NativePaddlePredictor::Run(std::vector<paddle::PaddleTensor, std::allocatorpaddle::PaddleTensor > const&, std::vector<paddle::PaddleTensor, std::allocatorpaddle::PaddleTensor >, int) + 273
8 0x4a0697p
9 0x4c44c8p
10 0x495a20p
11 0x594116p baidu::rpc::policy::ProcessHttpRequest(baidu::rpc::InputMessageBase) + 3686
12 0x4eae1ap baidu::rpc::ProcessInputMessage(void*) + 10
13 0x4ec27fp baidu::rpc::InputMessenger::OnNewMessages(baidu::rpc::Socket*) + 383
14 0x51518dp baidu::rpc::Socket::ProcessEvent(void*) + 13
15 0x67bc6ap bthread::TaskGroup::task_runner(long) + 266
16 0x6713a1p