You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
We have a rare crash in kamailio v5.3 running on Debian Buster in production environment. The crash (segmentation fault) happens while app_json module is writing accounting data to syslog upon receiving ACK for successful INVITE transaction over IPv6 UDP socket.
Troubleshooting
We checked the server health and service load at the time of crash, everything looks normal. We inspected the core dump generated by kamailio, and it seems that there is some sort of race condition causing the crash, the ACK completes the INVITE transaction, so kamailio was fetching the transaction data from memory to write accounting record to syslog, however, that data is corrupted or deleted by some other kamailio process. We do not see anything unusual in kamailio log related to crash. Perhaps it is related to this older but unresolved crash report.
Reproduction
No, we are unable to reproduce it in our lab.
Debugging Data
Output of bt full (first 16 function traces),
#0 0x0000561a11a47ec7 in match_by_name (avp=0xd, id=52, name=0x7ffccc7967f8) at core/usr_avp.c:379
avp_name = 0x7f10ead4f190#1 0x0000561a11a48d15 in search_next_avp (s=0x7ffccc7967f0, val=0x7ffccc7967d0) at core/usr_avp.c:500
matched = 0
avp = 0x7f10ead4f180 list = 0x7f10dc3196a8
__func__ = "search_next_avp"#2 0x0000561a11a486d1 in search_avp (ident=..., val=0x7ffccc7967d0, state=0x7ffccc7967f0) at core/usr_avp.c:466
ret = 0x7f10ead4f180
st = {flags = 273, id = 10, name = {n = -567873331, s = {s = 0x7f11de26f0cd"mapped_identity_user)", len = 20}, re = 0x7f11de26f0cd}, avp = 0x7f10f0114820}
list = 0x7f10f4d2b208
__func__ = "search_avp"#3 0x0000561a11a4804b in search_first_avp (flags=1, name=..., val=0x7ffccc7967d0, s=0x7ffccc7967f0) at core/usr_avp.c:415
id = {flags = 1, name = {n = -573917191, s = {s = 0x7f11ddcab7f9"from_mandant);tm", len = 12}, re = 0x7f11ddcab7f9}, index = 0}
#4 0x00007f11dcfb0392 in pv_get_avp (msg=0x7f11de70c4b0, param=0x7f11de6cf7b0, res=0x7ffccc796930) at pv_core.c:1793
name_type = 1
avp_name = {n = -573917191, s = {s = 0x7f11ddcab7f9"from_mandant);tm", len = 12}, re = 0x7f11ddcab7f9}
avp_value = {n = -563284264, s = {s = 0x7f11de6cf6d8"", len = -563034960}, re = 0x7f11de6cf6d8}
avp = 0x2000000
avp_value0 = {n = -864458672, s = {s = 0x7ffccc796850"\300hy\314\374\177", len = -587590620}, re = 0x7ffccc796850}
avp0 = 0x7f11dcd1bc90 <val_arr+336>
idx = 0
idxf = 0 p = 0x14d0960 <error: Cannot access memory at address0x14d0960>
p_ini = 0x8f6c38 <error: Cannot access memory at address0x8f6c38>
p_size = 22042 n = 0
state = {flags = 497, id = 52, name = {n = -573917191, s = {s = 0x7f11ddcab7f9"from_mandant);tm", len = 12}, re = 0x7f11ddcab7f9}, avp = 0xd}
__func__ = "pv_get_avp"#5 0x0000561a11a77f17 in pv_get_spec_value (msg=0x7f11de70c4b0, sp=0x7f11de6cf798, value=0x7ffccc796930) at core/pvapi.c:1404
ret = 0
__func__ = "pv_get_spec_value"#6 0x00007f11dcd0c43a in extra2strar (extra=0x7f11de6cf788, rq=0x7f11de70c4b0, val_arr=0x7f11dcd1bb90 <val_arr+80>, int_arr=0x7f11dcd1c4d4 <int_arr+20>,
type_arr=0x7f11dcd1c725 <type_arr+5> "\002\002\002\002\002\002\002\002\001") at acc_extra.c:222
value = {rs = {s = 0x0, len = 0}, ri = 0, flags = 0}
n = 18
i = 0
__func__ = "extra2strar"#7 0x00007f11dc2afc85 in acc_json_send_request (req=0x7f11de70c4b0, inf=0x7ffccc796af0) at acc_json_mod.c:308
attr_cnt = 5
i = 5
m = 0
o = 0
object = 0x561a13e7b7f0
__func__ = "acc_json_send_request"
extra = 0x561a11c6f34a#8 0x00007f11dcce5a84 in acc_run_engines (msg=0x7f11de70c4b0, type=0, reset=0x0) at acc.c:581
inf = {env = 0x7f11dcd423a0 <acc_env>, varr = 0x7f11dcd1bb40 <val_arr>, iarr = 0x7f11dcd1c4c0 <int_arr>, tarr = 0x7f11dcd1c720 <type_arr> "\002\002\002\002\001\002\002\002\002\002\002\002\002\001",
leg_info = 0x0}
e = 0x7f11de6cfa10
__func__ = "acc_run_engines"#9 0x00007f11dccf90a4 in acc_onack (t=0x7f10f4d2b008, req=0x7f10f52e2620, ack=0x7f11de70c4b0, code=-4) at acc_logic.c:657
__func__ = "acc_onack"#10 0x00007f11dccf959c in tmcb_func (t=0x7f10f4d2b008, type=4, ps=0x7ffccc796d10) at acc_logic.c:696
__func__ = "tmcb_func"#11 0x00007f11dd158e87 in run_trans_callbacks_internal (cb_lst=0x7f10f4d2b080, type=4, trans=0x7f10f4d2b008, params=0x7ffccc796d10) at t_hooks.c:254
cbp = 0x7f10f70688d8
backup_from = 0x561a11d83f30 <def_list+16>
backup_to = 0x561a11d83f38 <def_list+24>
backup_dom_from = 0x561a11d83f40 <def_list+32>
backup_dom_to = 0x561a11d83f48 <def_list+40>
backup_uri_from = 0x561a11d83f20 <def_list>
backup_uri_to = 0x561a11d83f28 <def_list+8>
backup_xavps = 0x561a11d16180 <_xavp_list_head>
__func__ = "run_trans_callbacks_internal"#12 0x00007f11dd158fae in run_trans_callbacks (type=4, trans=0x7f10f4d2b008, req=0x7f11de70c4b0, rpl=0x0, code=-4) at t_hooks.c:279
params = {req = 0x7f11de70c4b0, rpl = 0x0, param = 0x7f10f70688e8, code = -4, flags = 0, branch = 0, t_rbuf = 0x0, dst = 0x0, send_buf = {s = 0x0, len = 0}}
#13 0x00007f11dd0bd4b4 in t_newtran (p_msg=0x7f11de70c4b0) at t_lookup.c:1437
lret = -2
my_err = -562995536
canceled = 0
__func__ = "t_newtran"#14 0x00007f11dd14fb4b in t_relay_to (p_msg=0x7f11de70c4b0, proxy=0x0, proto=0, replicate=0) at t_funcs.c:243
ret = 0
new_tran = 32764
t = 0x7f11de70c4b0
dst = {send_sock = 0x561a11867b90 <_start>, to = {s = {sa_family = 43008, sa_data = "_V\004\271f\377\000\000\000\000\000\000\000"}, sin = {sin_family = 43008, sin_port = 22111, sin_addr = {
s_addr = 4284922116}, sin_zero = "\000\000\000\000\000\000\000"}, sin6 = {sin6_family = 43008, sin6_port = 22111, sin6_flowinfo = 4284922116, sin6_addr = {__in6_u = {
__u6_addr8 = '\000' <repeats 15 times>, __u6_addr16 = {0, 0, 0, 0, 0, 0, 0, 0}, __u6_addr32 = {0, 0, 0, 0}}}, sin6_scope_id = 294026128}, sas = {ss_family = 43008,
__ss_padding = "_V\004\271f\377", '\000' <repeats 16 times>, "\220{\206\021\032V\000\000\260^q\336\021\177\000\000H\221?\336\021\177\000\000\030\203\017\336\021\177", '\000' <repeats 14 times>, "\032V\000\000\000\000\000\002\000\000\000\000\000\250_V\004\271f\377\030e\217", '\000' <repeats 13 times>, "\220{\206\021\032V\000\000p\223y\314\374\177\000", __ss_align = 0}}, id = 0, send_flags = {f = 0,
blst_imask = 0}, proto = -128'\200', proto_pad0 = 111'o', proto_pad1 = -13191}
port = 0
host = {s = 0x2 <error: Cannot access memory at address0x2>, len = 0}
comp = 0
__func__ = "t_relay_to"#15 0x00007f11dd1737b4 in _w_t_relay_to (p_msg=0x7f11de70c4b0, proxy=0x0, force_proto=0) at tm.c:1691
t = 0x7f11de70c4b0
res = 0
__func__ = "_w_t_relay_to"#16 0x00007f11dd174aab in w_t_relay (p_msg=0x7f11de70c4b0, _foo=0x0, _bar=0x0) at tm.c:1891
No locals.
Nov2816:11:59 sip-proxy2 kamailio[1370]:ALERT: <core> [main.c:766]:handle_sigs(): child process 1596 exited by a signal 11Nov2816:11:59 sip-proxy2 kamailio[1370]:ALERT: <core> [main.c:769]:handle_sigs(): core was generated
Thanks for the report. As the 5.3.x branch is now end of life, it would be good if you can update to a more recent version, like one of the last version of the 5.5.x or 5.6.x branches. Maybe its already fixed in a newer version.
OK. Since it is a very rare issue (only happened twice this year), and we are already planning to upgrade kamailio, so, we can pause or close it for now and see what happens after upgrade.
Description
We have a rare crash in kamailio v5.3 running on Debian Buster in production environment. The crash (segmentation fault) happens while app_json module is writing accounting data to syslog upon receiving ACK for successful INVITE transaction over IPv6 UDP socket.
Troubleshooting
We checked the server health and service load at the time of crash, everything looks normal. We inspected the core dump generated by kamailio, and it seems that there is some sort of race condition causing the crash, the ACK completes the INVITE transaction, so kamailio was fetching the transaction data from memory to write accounting record to syslog, however, that data is corrupted or deleted by some other kamailio process. We do not see anything unusual in kamailio log related to crash. Perhaps it is related to this older but unresolved crash report.
Reproduction
No, we are unable to reproduce it in our lab.
Debugging Data
Output of
bt full
(first 16 function traces),Module parameters for
acc_json
,Log Messages
SIP Traffic
Possible Solutions
Nothing.
Additional Information
kamailio -v
The text was updated successfully, but these errors were encountered: