worker process crash

[zhangping]

#0  0x00007f8a7a80c6de in ?? () from /usr/lib/x86_64-linux-gnu/libvo-aacenc.so.0
#1  0x00007f8a7a80f23b in ?? () from /usr/lib/x86_64-linux-gnu/libvo-aacenc.so.0
#2  0x00007f8a7a8182f4 in ?? () from /usr/lib/x86_64-linux-gnu/libvo-aacenc.so.0
#3  0x00007f8a7a80d145 in ?? () from /usr/lib/x86_64-linux-gnu/libvo-aacenc.so.0
#4  0x00007f8a7a80c95f in ?? () from /usr/lib/x86_64-linux-gnu/libvo-aacenc.so.0
#5  0x00007f8a7aa2a4e2 in gst_voaacenc_handle_frame (benc=0xfa8680, buf=0x7f8a6c01c4c0) at gstvoaacenc.c:486
#6  0x00007f8a7c8ef58d in gst_audio_encoder_push_buffers (enc=enc@entry=0xfa8680, force=force@entry=0) at gstaudioencoder.c:978
#7  0x00007f8a7c8efafc in gst_audio_encoder_chain (pad=<optimized out>, parent=0xfa8680, buffer=0x7f8a6c012a40) at gstaudioencoder.c:1197
#8  0x00007f8a8983ad08 in gst_pad_chain_data_unchecked (data=0x7f8a6c012a40, type=4112, pad=0xfa4500) at gstpad.c:3760
#9  gst_pad_push_data (pad=0xfa42d0, type=type@entry=4112, data=<optimized out>) at gstpad.c:3990
#10 0x00007f8a898419b6 in gst_pad_push (pad=<optimized out>, buffer=<optimized out>) at gstpad.c:4093
#11 0x00007f8a880582a9 in gst_base_transform_chain (pad=<optimized out>, parent=0xfa16e0, buffer=<optimized out>) at gstbasetransform.c:2237
#12 0x00007f8a8983ad08 in gst_pad_chain_data_unchecked (data=0x7f8a6c01e8d0, type=4112, pad=0xfa40a0) at gstpad.c:3760
#13 gst_pad_push_data (pad=0xf91d80, type=type@entry=4112, data=<optimized out>, data@entry=0x7f8a6c01e8d0) at gstpad.c:3990
#14 0x00007f8a898419b6 in gst_pad_push (pad=<optimized out>, buffer=buffer@entry=0x7f8a6c01e8d0) at gstpad.c:4093
#15 0x00007f8a7d3c5eb0 in gst_queue_push_one (queue=0xf96260) at gstqueue.c:1115
#16 gst_queue_loop (pad=<optimized out>) at gstqueue.c:1244
#17 0x00007f8a89868549 in gst_task_func (task=0xfdd5f0) at gsttask.c:316
#18 0x00007f8a88d1c89c in ?? () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#19 0x00007f8a88d1bf15 in ?? () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#20 0x00007f8a88890182 in start_thread (arg=0x7f8a6affd700) at pthread_create.c:312
#21 0x00007f8a8837730d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

[zhangping]

Core was generated by `/usr/local/bin/gstreamill -l /var/log/gstreamill -n Q0NUVi00 -q 4154'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  __memcpy_sse2_unaligned () at ../sysdeps/x86_64/multiarch/memcpy-sse2-unaligned.S:152
152     ../sysdeps/x86_64/multiarch/memcpy-sse2-unaligned.S: No such file or directory.
(gdb) bt
#0  __memcpy_sse2_unaligned () at ../sysdeps/x86_64/multiarch/memcpy-sse2-unaligned.S:152
#1  0x00007f48ca4fe958 in memcpy (__len=18446744073709551570, __src=0x7f48b80209ae, __dest=<optimized out>)
    at /usr/include/x86_64-linux-gnu/bits/string3.h:51
#2  mpegts_packetizer_push_section (packetizer=packetizer@entry=0x1b9b040, packet=packet@entry=0x7f48c7966b30, remaining=remaining@entry=0x7f48c7966b28)
    at mpegtspacketizer.c:1030
#3  0x00007f48ca501f46 in mpegts_base_chain (pad=<optimized out>, parent=0x1b96100, buf=0x7f48b800c5f0) at mpegtsbase.c:1121
#4  0x00007f48d6ba6d08 in gst_pad_chain_data_unchecked (data=0x7f48b800c5f0, type=4112, pad=0x1b82730) at gstpad.c:3760
#5  gst_pad_push_data (pad=0x1b82500, type=type@entry=4112, data=<optimized out>, data@entry=0x7f48b800c5f0) at gstpad.c:3990
#6  0x00007f48d6bad9b6 in gst_pad_push (pad=<optimized out>, buffer=buffer@entry=0x7f48b800c5f0) at gstpad.c:4093
#7  0x00007f48ca731eb0 in gst_queue_push_one (queue=0x1b90200) at gstqueue.c:1115
#8  gst_queue_loop (pad=<optimized out>) at gstqueue.c:1244
#9  0x00007f48d6bd4549 in gst_task_func (task=0x1c2e050) at gsttask.c:316
#10 0x00007f48d608889c in ?? () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#11 0x00007f48d6087f15 in ?? () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#12 0x00007f48d5bfc182 in start_thread (arg=0x7f48c7967700) at pthread_create.c:312
#13 0x00007f48d56e330d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:111

[zhangping]

Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00007f3e22fa128a in g_slice_alloc () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
(gdb) bt
#0  0x00007f3e22fa128a in g_slice_alloc () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#1  0x00007f3e22fa4fd3 in g_string_sized_new () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#2  0x00007f3e22fa5623 in g_string_new () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#3  0x00007f3e22f8c49a in g_log_default_handler () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#4  0x00007f3e22f8cae1 in g_logv () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#5  0x00007f3e22f8cd72 in g_log () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#6  0x0000000000413dda in log_func (category=<optimized out>, level=<optimized out>, file=0x41e4b7 "source.c", function=<optimized out>, line=242,
    object=<optimized out>, message=0x7fff74d7f060, user_data=0x6c1660) at log.c:154
#7  0x00007f3e23abed3c in gst_debug_log_valist (category=0x7e5620, level=GST_LEVEL_INFO, file=0x41e4b7 "source.c",
    function=0x41eb0b <__FUNCTION__.19646> "bus_callback", line=242, object=0x0, format=format@entry=0x41e52b "%s message: %s",
    args=args@entry=0x7fff74d7f0e8) at gstinfo.c:509
#8  0x00007f3e23abeea7 in gst_debug_log (category=<optimized out>, level=level@entry=GST_LEVEL_INFO, file=file@entry=0x41e4b7 "source.c",
    function=function@entry=0x41eb0b <__FUNCTION__.19646> "bus_callback", line=<optimized out>, object=<optimized out>, format=0x41e52b "%s message: %s")
    at gstinfo.c:436
#9  0x000000000040d928 in bus_callback (bus=<optimized out>, msg=<optimized out>, user_data=0x7e51c0) at source.c:242
#10 0x00007f3e23aa3f71 in gst_bus_source_dispatch (source=0x8458a0, callback=0x40d830 <bus_callback>, user_data=0x7e51c0) at gstbus.c:769
#11 0x00007f3e22f85ce5 in g_main_context_dispatch () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#12 0x00007f3e22f86048 in ?? () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#13 0x00007f3e22f8630a in g_main_loop_run () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#14 0x0000000000407553 in main (argc=1, argv=0x7fff74d7f768) at main.c:367

[zhangping]

首次启动job处于playing状态，主进程在create_job_process函数中等待子进程状态处于sleep的时候，job进程exit 101，被watch进程收集到，此时create_job_process从sleep中出来返回failure，主进程得到启动失败的结果，于是释放job，而此时watch中开始重启这个job被释放了的job，必然会出问题。

Mar 21 16:46:28.286848 ERROR  gstreamill gstreamill.c:1156: sem_unlink 00000110240256_1 error: No such file or directory
Mar 21 16:46:28.575810 WARN   gstreamill gstreamill.c:1025: Start job failure: subprocess return 9 and don't restart it.
Mar 21 16:46:28.581776 WARN   gstreamill gstreamill.c:1186: Start job 00000110240256_1 failure, return stat: JOB_STATE_START_FAILURE
Mar 21 16:47:10.722320 ERROR  gstreamill gstreamill.c:698: cache_size:67108864, rap_addr:67070616, n:38236, 0x7f45b3510b40
Mar 21 16:47:20.851137 ERROR  gstreamill gstreamill.c:1156: sem_unlink 00000110240256_1 error: No such file or directory
Mar 21 16:47:21.264546 WARN   gstreamill gstreamill.c:1034: Job 00000110240256_1 exit on critical error return 101, restart ...
Mar 21 16:47:21.292062 WARN   gstreamill gstreamill.c:989: Read job 00000110240256_1's stat failure.
Mar 21 16:47:21.292122 WARN   gstreamill gstreamill.c:1186: Start job 00000110240256_1 failure, return stat: JOB_STATE_START_FAILURE

启动job的时候失败退出，exit(101)

[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `gstreamill -d -m 10.10.2.30:20118 -a 10.10.2.30:20119'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x000000000040a7c7 in create_job_process (job=job@entry=0x7f462c004e40) at gstreamill.c:984
984     gstreamill.c: No such file or directory.
(gdb) bt
#0  0x000000000040a7c7 in create_job_process (job=job@entry=0x7f462c004e40) at gstreamill.c:984
#1  0x000000000040ab0f in child_watch_cb (pid=4062, status=<optimized out>, job=0x7f462c004e40) at gstreamill.c:1036
#2  0x00007f46817f8b94 in ?? () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#3  0x00007f46817fbce5 in g_main_context_dispatch () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#4  0x00007f46817fc048 in ?? () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#5  0x00007f46817fc30a in g_main_loop_run () from /lib/x86_64-linux-gnu/libglib-2.0.so.0
#6  0x000000000040893b in main (argc=1, argv=0x7fffaa180cb8) at main.c:601

编码进程：

Mar 21 16:47:21.255926 FIXME  default gstutils.c:3766: Creating random stream-id, consider implementing a deterministic way of creating a stream-id
Mar 21 16:47:21.256009 WARN   queue gstqueue.c:1515: error: Internal data flow error.
Mar 21 16:47:21.256073 WARN   queue gstqueue.c:1515: error: streaming task paused, reason not-negotiated (-4)
Mar 21 16:47:21.256144 WARN   queue gstqueue.c:968: error: Internal data flow error.
Mar 21 16:47:21.256177 WARN   queue gstqueue.c:968: error: streaming task paused, reason not-negotiated (-4)
Mar 21 16:47:21.257093 WARN   collectpads gstcollectpads.c:1415: GstCollectPads has no time segment, assuming 0 based.
Mar 21 16:47:21.257215 WARN   mpegtsmux mpegtsmux.c:796: error: Could not create handler for stream
Mar 21 16:47:21.257401 WARN   queue gstqueue.c:1515: error: Internal data flow error.
Mar 21 16:47:21.257499 WARN   queue gstqueue.c:1515: error: streaming task paused, reason not-negotiated (-4)
Mar 21 16:47:21.257598 FIXME  default gstutils.c:3766: Creating random stream-id, consider implementing a deterministic way of creating a stream-id
Mar 21 16:47:21.257630 WARN   queue gstqueue.c:968: error: Internal data flow error.
Mar 21 16:47:21.257673 WARN   queue gstqueue.c:968: error: streaming task paused, reason not-negotiated (-4)
Mar 21 16:47:21.257802 FIXME  default gstutils.c:3766: Creating random stream-id, consider implementing a deterministic way of creating a stream-id

*** Mar 21 16:47:21 : job 00000110240256_1 started ***

Mar 21 16:47:21.258801 WARN   collectpads gstcollectpads.c:1415: GstCollectPads has no time segment, assuming 0 based.
Mar 21 16:47:21.258887 WARN   mpegtsmux mpegtsmux.c:796: error: Could not create handler for stream
Mar 21 16:47:21.259024 ERROR  gstreamill source.c:209: encoder.0 error found: Could not create handler for stream, exit