Nginx may crash, when reading request body in ngx.thread.spawn.

[Masterlvng]

Nginx Version:

nginx version: openresty/1.27.1.2
built by gcc 8.5.0 20210514 (TencentOS 8.5.0-18) (GCC) 
built with OpenSSL 1.1.1k  FIPS 25 Mar 2021
TLS SNI support enabled

Nginx.conf:

...
http {
...
access_by_lua_file /usr/local/openresty/lualib/access_spawn_panic.lua;
...
}
...

access_spawn_panic.lua:

local ngx = ngx
local socket = ngx.socket

local function parent_task()
    ngx.req.read_body()
    local tcp = socket.tcp()
    -- coredump at this line
    local ok, err = tcp:connect("127.0.0.1", 8090)
    if not ok then
        return
    end
    tcp:close()
end

local function child_task()
    ngx.req.read_body()
    return
end

local function main()
    local co = ngx.thread.spawn(child_task)
    parent_task()

    local ok, err = ngx.thread.wait(co)
    if not ok then
        ngx.log(ngx.ERR, "wait err ", err)
    end
end

main()

How to Reproduce

# Upload a file to trigger I/O interruptions when Lua reads the buffered request body.
curl -X POST -F "file=@body_5m.txt" http://127.0.0.1

Coredump Backtrace

Program received signal SIGSEGV, Segmentation fault.
ngx_http_lua_socket_resolve_retval_handler (r=r@entry=0x1964770, u=u@entry=0x7f0f3b514590, L=L@entry=0x7f0f3b5318a8)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_socket_tcp.c:1558
1558        ngx_http_lua_cleanup_pending_operation(coctx);
(gdb) bt
#0  ngx_http_lua_socket_resolve_retval_handler (r=r@entry=0x1964770, u=u@entry=0x7f0f3b514590, 
    L=L@entry=0x7f0f3b5318a8) at ../ngx_lua-0.10.28/src/ngx_http_lua_socket_tcp.c:1558
#1  0x00000000004efc74 in ngx_http_lua_socket_tcp_connect_helper (L=L@entry=0x7f0f3b5318a8, 
    u=u@entry=0x7f0f3b514590, r=r@entry=0x1964770, ctx=ctx@entry=0x19654d8, 
    host_ref=host_ref@entry=0x7f0f3b5432e8 "127.0.0.1", host_len=9, port=<optimized out>, resuming=<optimized out>)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_socket_tcp.c:716
#2  0x00000000004f08a9 in ngx_http_lua_socket_tcp_connect (L=0x7f0f3b5318a8)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_socket_tcp.c:1189
#3  0x00007f0f3accdbbb in lj_BC_FUNCC () from /usr/local/openresty/luajit/lib/libluajit-5.1.so.2
#4  0x00000000004e20f2 in ngx_http_lua_run_thread (L=L@entry=0x7f0f3b538380, r=r@entry=0x1964770, 
    ctx=ctx@entry=0x19654d8, nrets=<optimized out>, nrets@entry=0)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_util.c:1190
#5  0x00000000004e3b26 in ngx_http_lua_run_posted_threads (c=c@entry=0x7f0f3b4bd3b0, L=L@entry=0x7f0f3b538380, 
    r=r@entry=0x1964770, ctx=ctx@entry=0x19654d8, nreqs=nreqs@entry=1)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_util.c:3242
#6  0x00000000004e3ba5 in ngx_http_lua_run_posted_threads (c=c@entry=0x7f0f3b4bd3b0, L=L@entry=0x7f0f3b538380, 
    r=r@entry=0x1964770, ctx=ctx@entry=0x19654d8, nreqs=nreqs@entry=1)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_util.c:3217
#7  0x00000000004e6cc7 in ngx_http_lua_access_by_chunk (L=L@entry=0x7f0f3b538380, r=r@entry=0x1964770)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_accessby.c:359
#8  0x00000000004e723b in ngx_http_lua_access_handler_file (r=0x1964770)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_accessby.c:229
#9  0x00000000004e6e17 in ngx_http_lua_access_handler (r=0x1964770)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_accessby.c:158
--Type <RET> for more, q to quit, c to continue without paging--
#10 0x000000000045f47c in ngx_http_core_access_phase (r=0x1964770, ph=0x18fba98)
    at src/http/ngx_http_core_module.c:1112
#11 0x000000000045afd5 in ngx_http_core_run_phases (r=r@entry=0x1964770) at src/http/ngx_http_core_module.c:885
#12 0x000000000045b0a2 in ngx_http_handler (r=r@entry=0x1964770) at src/http/ngx_http_core_module.c:868
#13 0x0000000000465136 in ngx_http_process_request (r=r@entry=0x1964770) at src/http/ngx_http_request.c:2163
#14 0x0000000000465c3b in ngx_http_process_request_headers (rev=rev@entry=0x192c860)
    at src/http/ngx_http_request.c:1552
#15 0x0000000000465fd6 in ngx_http_process_request_line (rev=0x192c860) at src/http/ngx_http_request.c:1219
#16 0x000000000044d7ee in ngx_epoll_process_events (cycle=<optimized out>, timer=<optimized out>, flags=1)
    at src/event/modules/ngx_epoll_module.c:901
#17 0x0000000000444b53 in ngx_process_events_and_timers (cycle=cycle@entry=0x18d2b30) at src/event/ngx_event.c:258
#18 0x000000000044bbea in ngx_worker_process_cycle (cycle=0x18d2b30, data=<optimized out>)
    at src/os/unix/ngx_process_cycle.c:793
#19 0x000000000044a624 in ngx_spawn_process (cycle=cycle@entry=0x18d2b30, proc=0x44bb70 <ngx_worker_process_cycle>, 
    data=0x0, name=0x544145 "worker process", respawn=respawn@entry=0) at src/os/unix/ngx_process.c:207
#20 0x000000000044cbc1 in ngx_reap_children (cycle=0x18d2b30) at src/os/unix/ngx_process_cycle.c:665
#21 ngx_master_process_cycle (cycle=0x18d2b30) at src/os/unix/ngx_process_cycle.c:180
#22 0x0000000000424152 in main (argc=<optimized out>, argv=<optimized out>) at src/core/nginx.c:387

Why coredump

1, When spawn thread end, ctx->cur_co_ctx is assigned to null

ngx_http_lua_util.c#L1395-L1425

            if (ctx->cur_co_ctx->is_uthread) {
                ngx_http_lua_assert(err != NULL && msg != NULL
                                    && trace != NULL);

                ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
                              "lua user thread aborted: %s: %s\n%s",
                              err, msg, trace);

                lua_settop(L, 0);

                parent_coctx = ctx->cur_co_ctx->parent_co_ctx;

                if (ngx_http_lua_coroutine_alive(parent_coctx)) {
                    if (ctx->cur_co_ctx->waited_by_parent) {
                        ctx->cur_co_ctx->waited_by_parent = 0;
                        success = 0;
                        goto user_co_done;
                    }

                    if (ngx_http_lua_post_zombie_thread(r, parent_coctx,
                                                        ctx->cur_co_ctx)
                        != NGX_OK)
                    {
                        return NGX_ERROR;
                    }

                    lua_pushboolean(ctx->cur_co_ctx->co, 0);
                    lua_insert(ctx->cur_co_ctx->co, 1);

                    ctx->cur_co_ctx->co_status = NGX_HTTP_LUA_CO_ZOMBIE;
                    // 
                    ctx->cur_co_ctx = NULL;
                    return NGX_AGAIN;
                }

2, When the parent coroutine comes back after reading the body, it doesn't restore cur_co_ctx correctly.

ngx_http_lua_req_body.c#L1395-L1425

    if (rc == NGX_AGAIN) {
        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                       "lua read buffered request body requires I/O "
                       "interruptions");


        ctx->waiting_more_body = 1;
        ctx->downstream = coctx;


        ngx_http_lua_cleanup_pending_operation(coctx);
        coctx->cleanup = ngx_http_lua_req_body_cleanup;
        coctx->data = r;


        return lua_yield(L, 0);
    }


    /* rc == NGX_OK */


    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                   "lua has read buffered request body in a single run");

   /*  do something, restore co ctx? */
    return 0;
}

3, When parent coroutine call tcp:connect, it coredump, because cur_co_ctx is NULL

#0  ngx_http_lua_socket_resolve_retval_handler (r=r@entry=0x1964770, u=u@entry=0x7f0f3b514590, 
    L=L@entry=0x7f0f3b5318a8) at ../ngx_lua-0.10.28/src/ngx_http_lua_socket_tcp.c:1558
#1  0x00000000004efc74 in ngx_http_lua_socket_tcp_connect_helper (L=L@entry=0x7f0f3b5318a8, 
    u=u@entry=0x7f0f3b514590, r=r@entry=0x1964770, ctx=ctx@entry=0x19654d8, 
    host_ref=host_ref@entry=0x7f0f3b5432e8 "127.0.0.1", host_len=9, port=<optimized out>, resuming=<optimized out>)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_socket_tcp.c:716
#2  0x00000000004f08a9 in ngx_http_lua_socket_tcp_connect (L=0x7f0f3b5318a8)
    at ../ngx_lua-0.10.28/src/ngx_http_lua_socket_tcp.c:1189

If you need any additional information, please let me know. : )

[zhuizhuhaomeng]

See this PR: #2451