Skip to content

Commit 27e9498

Browse files
author
mdounin
committed
Upstream: smooth weighted round-robin balancing.
For edge case weights like { 5, 1, 1 } we now produce { a, a, b, a, c, a, a } sequence instead of { c, b, a, a, a, a, a } produced previously. Algorithm is as follows: on each peer selection we increase current_weight of each eligible peer by its weight, select peer with greatest current_weight and reduce its current_weight by total number of weight points distributed among peers. In case of { 5, 1, 1 } weights this gives the following sequence of current_weight's: a b c 0 0 0 (initial state) 5 1 1 (a selected) -2 1 1 3 2 2 (a selected) -4 2 2 1 3 3 (b selected) 1 -4 3 6 -3 4 (a selected) -1 -3 4 4 -2 5 (c selected) 4 -2 -2 9 -1 -1 (a selected) 2 -1 -1 7 0 0 (a selected) 0 0 0 To preserve weight reduction in case of failures the effective_weight variable was introduced, which usually matches peer's weight, but is reduced temporarily on peer failures. This change also fixes loop with backup servers and proxy_next_upstream http_404 (ticket #47), and skipping alive upstreams in some cases if there are multiple dead ones (ticket #64). git-svn-id: svn://svn.nginx.org/nginx/trunk@4622 73f98a42-aea0-e011-b76d-00259023448c
1 parent 35b553d commit 27e9498

File tree

2 files changed

+72
-169
lines changed

2 files changed

+72
-169
lines changed

‎src/http/ngx_http_upstream_round_robin.c‎

Lines changed: 71 additions & 169 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@
1212

1313
static ngx_int_t ngx_http_upstream_cmp_servers(const void *one,
1414
const void *two);
15-
static ngx_uint_t
16-
ngx_http_upstream_get_peer(ngx_http_upstream_rr_peers_t *peers);
15+
static ngx_http_upstream_rr_peer_t *ngx_http_upstream_get_peer(
16+
ngx_http_upstream_rr_peer_data_t *rrp);
1717

1818
#if (NGX_HTTP_SSL)
1919

@@ -81,7 +81,8 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
8181
peers->peer[n].fail_timeout = server[i].fail_timeout;
8282
peers->peer[n].down = server[i].down;
8383
peers->peer[n].weight = server[i].down ? 0 : server[i].weight;
84-
peers->peer[n].current_weight = peers->peer[n].weight;
84+
peers->peer[n].effective_weight = peers->peer[n].weight;
85+
peers->peer[n].current_weight = 0;
8586
n++;
8687
}
8788
}
@@ -131,7 +132,8 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
131132
backup->peer[n].socklen = server[i].addrs[j].socklen;
132133
backup->peer[n].name = server[i].addrs[j].name;
133134
backup->peer[n].weight = server[i].weight;
134-
backup->peer[n].current_weight = server[i].weight;
135+
backup->peer[n].effective_weight = server[i].weight;
136+
backup->peer[n].current_weight = 0;
135137
backup->peer[n].max_fails = server[i].max_fails;
136138
backup->peer[n].fail_timeout = server[i].fail_timeout;
137139
backup->peer[n].down = server[i].down;
@@ -190,7 +192,8 @@ ngx_http_upstream_init_round_robin(ngx_conf_t *cf,
190192
peers->peer[i].socklen = u.addrs[i].socklen;
191193
peers->peer[i].name = u.addrs[i].name;
192194
peers->peer[i].weight = 1;
193-
peers->peer[i].current_weight = 1;
195+
peers->peer[i].effective_weight = 1;
196+
peers->peer[i].current_weight = 0;
194197
peers->peer[i].max_fails = 1;
195198
peers->peer[i].fail_timeout = 10;
196199
}
@@ -306,7 +309,8 @@ ngx_http_upstream_create_round_robin_peer(ngx_http_request_t *r,
306309
peers->peer[0].socklen = ur->socklen;
307310
peers->peer[0].name = ur->host;
308311
peers->peer[0].weight = 1;
309-
peers->peer[0].current_weight = 1;
312+
peers->peer[0].effective_weight = 1;
313+
peers->peer[0].current_weight = 0;
310314
peers->peer[0].max_fails = 1;
311315
peers->peer[0].fail_timeout = 10;
312316

@@ -338,7 +342,8 @@ ngx_http_upstream_create_round_robin_peer(ngx_http_request_t *r,
338342
peers->peer[i].name.len = len;
339343
peers->peer[i].name.data = p;
340344
peers->peer[i].weight = 1;
341-
peers->peer[i].current_weight = 1;
345+
peers->peer[i].effective_weight = 1;
346+
peers->peer[i].current_weight = 0;
342347
peers->peer[i].max_fails = 1;
343348
peers->peer[i].fail_timeout = 10;
344349
}
@@ -378,8 +383,6 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
378383
{
379384
ngx_http_upstream_rr_peer_data_t *rrp = data;
380385

381-
time_t now;
382-
uintptr_t m;
383386
ngx_int_t rc;
384387
ngx_uint_t i, n;
385388
ngx_connection_t *c;
@@ -389,8 +392,6 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
389392
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
390393
"get rr peer, try: %ui", pc->tries);
391394

392-
now = ngx_time();
393-
394395
/* ngx_lock_mutex(rrp->peers->mutex); */
395396

396397
if (rrp->peers->last_cached) {
@@ -423,118 +424,15 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
423424

424425
/* there are several peers */
425426

426-
if (pc->tries == rrp->peers->number) {
427-
428-
/* it's a first try - get a current peer */
429-
430-
i = pc->tries;
431-
432-
for ( ;; ) {
433-
rrp->current = ngx_http_upstream_get_peer(rrp->peers);
434-
435-
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
436-
"get rr peer, current: %ui %i",
437-
rrp->current,
438-
rrp->peers->peer[rrp->current].current_weight);
439-
440-
n = rrp->current / (8 * sizeof(uintptr_t));
441-
m = (uintptr_t) 1 << rrp->current % (8 * sizeof(uintptr_t));
442-
443-
if (!(rrp->tried[n] & m)) {
444-
peer = &rrp->peers->peer[rrp->current];
445-
446-
if (!peer->down) {
447-
448-
if (peer->max_fails == 0
449-
|| peer->fails < peer->max_fails)
450-
{
451-
break;
452-
}
427+
peer = ngx_http_upstream_get_peer(rrp);
453428

454-
if (now - peer->checked > peer->fail_timeout) {
455-
peer->checked = now;
456-
break;
457-
}
458-
459-
peer->current_weight = 0;
460-
461-
} else {
462-
rrp->tried[n] |= m;
463-
}
464-
465-
pc->tries--;
466-
}
467-
468-
if (pc->tries == 0) {
469-
goto failed;
470-
}
471-
472-
if (--i == 0) {
473-
ngx_log_error(NGX_LOG_ALERT, pc->log, 0,
474-
"round robin upstream stuck on %ui tries",
475-
pc->tries);
476-
goto failed;
477-
}
478-
}
479-
480-
peer->current_weight--;
481-
482-
} else {
483-
484-
i = pc->tries;
485-
486-
for ( ;; ) {
487-
n = rrp->current / (8 * sizeof(uintptr_t));
488-
m = (uintptr_t) 1 << rrp->current % (8 * sizeof(uintptr_t));
489-
490-
if (!(rrp->tried[n] & m)) {
491-
492-
peer = &rrp->peers->peer[rrp->current];
493-
494-
if (!peer->down) {
495-
496-
if (peer->max_fails == 0
497-
|| peer->fails < peer->max_fails)
498-
{
499-
break;
500-
}
501-
502-
if (now - peer->checked > peer->fail_timeout) {
503-
peer->checked = now;
504-
break;
505-
}
506-
507-
peer->current_weight = 0;
508-
509-
} else {
510-
rrp->tried[n] |= m;
511-
}
512-
513-
pc->tries--;
514-
}
515-
516-
rrp->current++;
517-
518-
if (rrp->current >= rrp->peers->number) {
519-
rrp->current = 0;
520-
}
521-
522-
if (pc->tries == 0) {
523-
goto failed;
524-
}
525-
526-
if (--i == 0) {
527-
ngx_log_error(NGX_LOG_ALERT, pc->log, 0,
528-
"round robin upstream stuck on %ui tries",
529-
pc->tries);
530-
goto failed;
531-
}
532-
}
533-
534-
peer->current_weight--;
429+
if (peer == NULL) {
430+
goto failed;
535431
}
536432

537-
rrp->tried[n] |= m;
433+
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
434+
"get rr peer, current: %ui %i",
435+
rrp->current, peer->current_weight);
538436
}
539437

540438
pc->sockaddr = peer->sockaddr;
@@ -545,11 +443,6 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
545443

546444
if (pc->tries == 1 && rrp->peers->next) {
547445
pc->tries += rrp->peers->next->number;
548-
549-
n = rrp->peers->next->number / (8 * sizeof(uintptr_t)) + 1;
550-
for (i = 0; i < n; i++) {
551-
rrp->tried[i] = 0;
552-
}
553446
}
554447

555448
return NGX_OK;
@@ -595,56 +488,71 @@ ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
595488
}
596489

597490

598-
static ngx_uint_t
599-
ngx_http_upstream_get_peer(ngx_http_upstream_rr_peers_t *peers)
491+
static ngx_http_upstream_rr_peer_t *
492+
ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp)
600493
{
601-
ngx_uint_t i, n, reset = 0;
602-
ngx_http_upstream_rr_peer_t *peer;
494+
time_t now;
495+
uintptr_t m;
496+
ngx_int_t total;
497+
ngx_uint_t i, n;
498+
ngx_http_upstream_rr_peer_t *peer, *best;
603499

604-
peer = &peers->peer[0];
500+
now = ngx_time();
605501

606-
for ( ;; ) {
502+
best = NULL;
503+
total = 0;
607504

608-
for (i = 0; i < peers->number; i++) {
505+
for (i = 0; i < rrp->peers->number; i++) {
609506

610-
if (peer[i].current_weight <= 0) {
611-
continue;
612-
}
507+
n = i / (8 * sizeof(uintptr_t));
508+
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
613509

614-
n = i;
615-
616-
while (i < peers->number - 1) {
617-
618-
i++;
510+
if (rrp->tried[n] & m) {
511+
continue;
512+
}
619513

620-
if (peer[i].current_weight <= 0) {
621-
continue;
622-
}
514+
peer = &rrp->peers->peer[i];
623515

624-
if (peer[n].current_weight * 1000 / peer[i].current_weight
625-
> peer[n].weight * 1000 / peer[i].weight)
626-
{
627-
return n;
628-
}
516+
if (peer->down) {
517+
continue;
518+
}
629519

630-
n = i;
631-
}
520+
if (peer->max_fails
521+
&& peer->fails >= peer->max_fails
522+
&& now - peer->checked <= peer->fail_timeout)
523+
{
524+
continue;
525+
}
632526

633-
if (peer[i].current_weight > 0) {
634-
n = i;
635-
}
527+
peer->current_weight += peer->effective_weight;
528+
total += peer->effective_weight;
636529

637-
return n;
530+
if (peer->effective_weight < peer->weight) {
531+
peer->effective_weight++;
638532
}
639533

640-
if (reset++) {
641-
return 0;
534+
if (best == NULL || peer->current_weight > best->current_weight) {
535+
best = peer;
642536
}
537+
}
643538

644-
for (i = 0; i < peers->number; i++) {
645-
peer[i].current_weight = peer[i].weight;
646-
}
539+
if (best == NULL) {
540+
return NULL;
647541
}
542+
543+
i = best - &rrp->peers->peer[0];
544+
545+
rrp->current = i;
546+
547+
n = i / (8 * sizeof(uintptr_t));
548+
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
549+
550+
rrp->tried[n] |= m;
551+
552+
best->current_weight -= total;
553+
best->checked = now;
554+
555+
return best;
648556
}
649557

650558

@@ -683,15 +591,15 @@ ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
683591
peer->checked = now;
684592

685593
if (peer->max_fails) {
686-
peer->current_weight -= peer->weight / peer->max_fails;
594+
peer->effective_weight -= peer->weight / peer->max_fails;
687595
}
688596

689597
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
690598
"free rr peer failed: %ui %i",
691-
rrp->current, peer->current_weight);
599+
rrp->current, peer->effective_weight);
692600

693-
if (peer->current_weight < 0) {
694-
peer->current_weight = 0;
601+
if (peer->effective_weight < 0) {
602+
peer->effective_weight = 0;
695603
}
696604

697605
/* ngx_unlock_mutex(rrp->peers->mutex); */
@@ -705,12 +613,6 @@ ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
705613
}
706614
}
707615

708-
rrp->current++;
709-
710-
if (rrp->current >= rrp->peers->number) {
711-
rrp->current = 0;
712-
}
713-
714616
if (pc->tries) {
715617
pc->tries--;
716618
}

‎src/http/ngx_http_upstream_round_robin.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ typedef struct {
2020
ngx_str_t name;
2121

2222
ngx_int_t current_weight;
23+
ngx_int_t effective_weight;
2324
ngx_int_t weight;
2425

2526
ngx_uint_t fails;

0 commit comments

Comments
 (0)