From b6f7067ae226766e2abc3ab2a8612dc9c39bf4aa Mon Sep 17 00:00:00 2001 From: finditfun <44823100+finditfun@users.noreply.github.com> Date: Tue, 23 May 2023 00:14:57 +0800 Subject: [PATCH 1/4] Update ngx_http_dyups_module.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 问题： 在nginx中保存ip数量在10000左右，且开启健康检查情况下，当通过dyups更新一个ip数量在1300以上的upstream情况下，根据测试，发现每个进程大约会阻塞200ms，若机器有50+个worker进程，进程间阻塞加锁更新的方式将会导致正常请求得不到处理。 解决方式： 通过添加串行更新指令，让nginx的每个进程串行更新upstream，当每个进程更新完成后，对原子变量进行+1操作，这样下一个进程才会去更新。 进程间的串行更新，避免了多进程同时阻塞去更新upstream。 --- .../ngx_http_dyups_module.c | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c b/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c index 8135ad4336..6b11ae8238 100644 --- a/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c +++ b/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c @@ -36,6 +36,7 @@ typedef struct { typedef struct { ngx_flag_t enable; ngx_flag_t trylock; + ngx_flag_t serial_update; ngx_array_t dy_upstreams;/* ngx_http_dyups_srv_conf_t */ ngx_str_t shm_name; ngx_uint_t shm_size; @@ -72,6 +73,7 @@ typedef struct ngx_dyups_shctx_s { ngx_queue_t msg_queue; ngx_uint_t version; ngx_dyups_status_t *status; + ngx_atomic_uint_t flag; } ngx_dyups_shctx_t; @@ -79,6 +81,7 @@ typedef struct ngx_dyups_global_ctx_s { ngx_event_t msg_timer; ngx_slab_pool_t *shpool; ngx_dyups_shctx_t *sh; + ngx_int_t workers; } ngx_dyups_global_ctx_t; @@ -216,6 +219,13 @@ static ngx_command_t ngx_http_dyups_commands[] = { NGX_HTTP_MAIN_CONF_OFFSET, offsetof(ngx_http_dyups_main_conf_t, trylock), NULL }, + + { ngx_string("dyups_serial_update_upstream"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_conf_set_flag_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_dyups_main_conf_t, serial_update), + NULL }, ngx_null_command }; @@ -318,6 +328,7 @@ ngx_http_dyups_create_main_conf(ngx_conf_t *cf) dmcf->read_msg_timeout = NGX_CONF_UNSET_MSEC; dmcf->read_msg_log = NGX_CONF_UNSET; dmcf->trylock = NGX_CONF_UNSET; + dmcf->serial_update = NGX_CONF_UNSET; return dmcf; } @@ -349,6 +360,10 @@ ngx_http_dyups_init_main_conf(ngx_conf_t *cf, void *conf) if (dmcf->shm_size == NGX_CONF_UNSET_UINT) { dmcf->shm_size = 2 * 1024 * 1024; } + + if (dmcf->serial_update == NGX_CONF_UNSET) { + dmcf->serial_update = 0; + } return ngx_http_dyups_init_shm(cf, conf); } @@ -427,6 +442,7 @@ ngx_http_dyups_init_shm_zone(ngx_shm_zone_t *shm_zone, void *data) sh->version = 0; sh->status = NULL; + sh->flag = 0; return NGX_OK; } @@ -562,6 +578,8 @@ ngx_http_dyups_init_process(ngx_cycle_t *cycle) } ngx_http_dyups_api_enable = 1; + + ngx_dyups_global_ctx.workers = ccf->worker_processes; timer = &ngx_dyups_global_ctx.msg_timer; ngx_memzero(timer, sizeof(ngx_event_t)); @@ -2063,9 +2081,13 @@ ngx_http_dyups_read_msg(ngx_event_t *ev) ngx_slab_pool_t *shpool; ngx_http_dyups_srv_conf_t *duscfs, *duscf; ngx_http_dyups_main_conf_t *dmcf; + ngx_dyups_shctx_t *sh; + ngx_uint_t workers; dmcf = ev->data; shpool = ngx_dyups_global_ctx.shpool; + sh = ngx_dyups_global_ctx.sh; + workers = ngx_dyups_global_ctx.workers; count = 0; s_count = 0; @@ -2096,7 +2118,13 @@ ngx_http_dyups_read_msg(ngx_event_t *ev) } #if (NGX_HTTP_UPSTREAM_CHECK) - if (!ngx_shmtx_trylock(&shpool->mutex)) { + if (dmcf->serial_update && (sh->flag % workers != ngx_worker)) { + ngx_log_error(NGX_LOG_DEBUG, ev->log, 0, + "[dyups] ngx_pid: %P, ngx_worker: %d, sh->flag: %d, not meet lock condition", ngx_pid, ngx_worker, sh->flag); + ngx_dyups_add_timer(ev, dmcf->read_msg_timeout); + return; + } + else if (!ngx_shmtx_trylock(&shpool->mutex)) { goto finish; } #else @@ -2108,6 +2136,14 @@ ngx_http_dyups_read_msg(ngx_event_t *ev) ngx_shmtx_unlock(&shpool->mutex); #if (NGX_HTTP_UPSTREAM_CHECK) + if (dmcf->serial_update) { + ngx_atomic_fetch_add(&sh->flag, 1); + + ngx_atomic_cmp_set(&sh->flag, workers, 0); + + ngx_log_error(NGX_LOG_DEBUG, ev->log, 0, + "[dyups] ngx_pid: %P, ngx_worker: %d, sh->flag: %d, finish read msg under locking condition", ngx_pid, ngx_worker, sh->flag); + } finish: #endif ngx_dyups_add_timer(ev, dmcf->read_msg_timeout); @@ -2129,6 +2165,7 @@ ngx_http_dyups_read_msg_locked(ngx_event_t *ev) ngx_dyups_msg_t *msg; ngx_dyups_shctx_t *sh; ngx_dyups_status_t *status; + ngx_http_dyups_main_conf_t *dmcf ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ev->log, 0, "[dyups] read msg %P", ngx_pid); @@ -2136,6 +2173,7 @@ ngx_http_dyups_read_msg_locked(ngx_event_t *ev) ccf = (ngx_core_conf_t *) ngx_get_conf(ngx_cycle->conf_ctx, ngx_core_module); + dmcf = ev->data; sh = ngx_dyups_global_ctx.sh; shpool = ngx_dyups_global_ctx.shpool; From a4b9ccd47559439f608d5abcafcceb0fa4fcb231 Mon Sep 17 00:00:00 2001 From: finditfun <44823100+finditfun@users.noreply.github.com> Date: Tue, 23 May 2023 10:02:49 +0800 Subject: [PATCH 2/4] Update ngx_http_dyups_module.c --- .../ngx_http_dyups_module.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c b/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c index 6b11ae8238..db746ed9ee 100644 --- a/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c +++ b/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c @@ -2118,11 +2118,13 @@ ngx_http_dyups_read_msg(ngx_event_t *ev) } #if (NGX_HTTP_UPSTREAM_CHECK) - if (dmcf->serial_update && (sh->flag % workers != ngx_worker)) { - ngx_log_error(NGX_LOG_DEBUG, ev->log, 0, + if (dmcf->serial_update) { + if (sh->flag % workers != ngx_worker) { + ngx_log_error(NGX_LOG_DEBUG, ev->log, 0, "[dyups] ngx_pid: %P, ngx_worker: %d, sh->flag: %d, not meet lock condition", ngx_pid, ngx_worker, sh->flag); - ngx_dyups_add_timer(ev, dmcf->read_msg_timeout); - return; + ngx_dyups_add_timer(ev, dmcf->read_msg_timeout); + return; + } } else if (!ngx_shmtx_trylock(&shpool->mutex)) { goto finish; From 38d26667ceb3340f7919ec4db44778ed4a7d29d1 Mon Sep 17 00:00:00 2001 From: finditfun <44823100+finditfun@users.noreply.github.com> Date: Tue, 23 May 2023 10:08:03 +0800 Subject: [PATCH 3/4] Update ngx_http_dyups_module.c --- modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c b/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c index db746ed9ee..d29b958cae 100644 --- a/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c +++ b/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c @@ -2125,6 +2125,7 @@ ngx_http_dyups_read_msg(ngx_event_t *ev) ngx_dyups_add_timer(ev, dmcf->read_msg_timeout); return; } + ngx_shmtx_lock(&shpool->mutex); } else if (!ngx_shmtx_trylock(&shpool->mutex)) { goto finish; From e012906e0b605c9894658663d242dd3d9ff7ef60 Mon Sep 17 00:00:00 2001 From: finditfun <44823100+finditfun@users.noreply.github.com> Date: Mon, 29 May 2023 10:20:54 +0800 Subject: [PATCH 4/4] fix compilation errors in dyups module fix a compiler bug caused by missing symbols --- modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c b/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c index d29b958cae..0b6f632464 100644 --- a/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c +++ b/modules/ngx_http_upstream_dyups_module/ngx_http_dyups_module.c @@ -2168,7 +2168,7 @@ ngx_http_dyups_read_msg_locked(ngx_event_t *ev) ngx_dyups_msg_t *msg; ngx_dyups_shctx_t *sh; ngx_dyups_status_t *status; - ngx_http_dyups_main_conf_t *dmcf + ngx_http_dyups_main_conf_t *dmcf; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, ev->log, 0, "[dyups] read msg %P", ngx_pid);