Permalink
Browse files

Support for servers marked "down".

Thanks to Jack Lindamood/Facebook.
  • Loading branch information...
1 parent 99ace64 commit fdaa270d3ba567a18ef49c470c3f48e0a9de7ed1 @evanmiller committed Jun 3, 2010
Showing with 69 additions and 29 deletions.
  1. +5 −0 CHANGES
  2. +3 −1 CREDITS
  3. +1 −0 config
  4. +3 −1 nginx.patch
  5. +57 −27 ngx_http_upstream_hash_module.c
View
@@ -1,3 +1,8 @@
+Changes with upstream_hash 0.3.2 03 Jun 2010
+
+ *) Support for servers marked as down. Thanks to Jack Lindamood of Facebook.
+
+
Changes with upstream_hash 0.3.1 19 Oct 2009
*) Update nginx.patch to work with more recent versions of nginx
View
@@ -1,4 +1,6 @@
-All development (so far) by Evan Miller.
+Most development (so far) by Evan Miller.
PECL Memcache compatibility sponsored by Spil Games, Inc.
[http://www.spilgames.com/]
+
+Support for servers marked as "down" contributed by Jack Lindamood of Facebook.
View
1 config
@@ -1,3 +1,4 @@
ngx_addon_name=ngx_http_upstream_hash_module
HTTP_MODULES="$HTTP_MODULES ngx_http_upstream_hash_module"
NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/ngx_http_upstream_hash_module.c"
+have=NGX_HTTP_UPSTREAM_HASH . auto/have
View
@@ -1,13 +1,15 @@
diff -Naur src/http/ngx_http_upstream.h src-hash/http/ngx_http_upstream.h
--- src/http/ngx_http_upstream.h 2009-09-07 04:49:51.000000000 -0500
+++ src-hash/http/ngx_http_upstream.h 2009-10-19 08:27:40.000000000 -0500
-@@ -105,6 +105,10 @@
+@@ -105,6 +105,12 @@
ngx_array_t *servers; /* ngx_http_upstream_server_t */
++#if (NGX_HTTP_UPSTREAM_HASH)
+ ngx_array_t *values;
+ ngx_array_t *lengths;
+ ngx_uint_t retries;
++#endif
+
ngx_uint_t flags;
ngx_str_t host;
@@ -1,6 +1,6 @@
/*
* Hash a variable to choose an upstream server.
- *
+ *
* Copyright (C) Evan Miller
*
* This module can be distributed under the same terms as Nginx itself.
@@ -11,20 +11,27 @@
#include <ngx_core.h>
#include <ngx_http.h>
-#define ngx_bitvector_index(index) index / (8 * sizeof(uintptr_t))
-#define ngx_bitvector_bit(index) (uintptr_t) 1 << index % (8 * sizeof(uintptr_t))
+#define ngx_bitvector_index(index) (index / (8 * sizeof(uintptr_t)))
+#define ngx_bitvector_bit(index) ((uintptr_t) 1 << (index % (8 * sizeof(uintptr_t))))
typedef struct {
struct sockaddr *sockaddr;
socklen_t socklen;
ngx_str_t name;
+ unsigned down:1;
} ngx_http_upstream_hash_peer_t;
typedef struct {
ngx_uint_t number;
ngx_http_upstream_hash_peer_t peer[0];
} ngx_http_upstream_hash_peers_t;
+/* The size of "hash" depends on the architecture (32-bit or 64-bit) to
+ * preserve bug-compatibility with PECL Memcache. Note that since the hash
+ * function produces 15-bit numbers, and the hashes are added up upon retry,
+ * achieving an overflow on a 32-bit machine requires ~100K retries. So in
+ * practice the architecture difference does not matter.
+ */
typedef struct {
ngx_http_upstream_hash_peers_t *peers;
ngx_uint_t hash;
@@ -35,6 +42,8 @@ typedef struct {
} ngx_http_upstream_hash_peer_data_t;
+static void ngx_http_upstream_hash_next_peer(ngx_http_upstream_hash_peer_data_t *uhpd,
+ ngx_uint_t *tries, ngx_log_t *log);
static ngx_int_t ngx_http_upstream_init_hash_peer(ngx_http_request_t *r,
ngx_http_upstream_srv_conf_t *us);
static ngx_int_t ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc,
@@ -43,9 +52,9 @@ static void ngx_http_upstream_free_hash_peer(ngx_peer_connection_t *pc,
void *data, ngx_uint_t state);
static char *ngx_http_upstream_hash(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
-static char *ngx_http_upstream_hash_again(ngx_conf_t *cf, ngx_command_t *cmd,
+static char *ngx_http_upstream_hash_again(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
-static ngx_int_t ngx_http_upstream_init_hash(ngx_conf_t *cf,
+static ngx_int_t ngx_http_upstream_init_hash(ngx_conf_t *cf,
ngx_http_upstream_srv_conf_t *us);
static ngx_uint_t ngx_http_upstream_hash_crc32(u_char *keydata, size_t keylen);
@@ -135,6 +144,7 @@ ngx_http_upstream_init_hash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us)
peers->peer[n].sockaddr = server[i].addrs[j].sockaddr;
peers->peer[n].socklen = server[i].addrs[j].socklen;
peers->peer[n].name = server[i].addrs[j].name;
+ peers->peer[n].down = server[i].down;
}
}
@@ -149,16 +159,16 @@ ngx_http_upstream_init_hash_peer(ngx_http_request_t *r,
ngx_http_upstream_srv_conf_t *us)
{
ngx_http_upstream_hash_peer_data_t *uhpd;
-
+
ngx_str_t val;
if (ngx_http_script_run(r, &val, us->lengths, 0, us->values) == NULL) {
return NGX_ERROR;
}
uhpd = ngx_pcalloc(r->pool, sizeof(ngx_http_upstream_hash_peer_data_t)
- + sizeof(uintptr_t) *
- ((ngx_http_upstream_hash_peers_t *)us->peer.data)->number /
+ + sizeof(uintptr_t) *
+ ((ngx_http_upstream_hash_peers_t *)us->peer.data)->number /
(8 * sizeof(uintptr_t)));
if (uhpd == NULL) {
return NGX_ERROR;
@@ -180,11 +190,17 @@ ngx_http_upstream_init_hash_peer(ngx_http_request_t *r,
ngx_memcpy(uhpd->current_key.data, val.data, val.len);
uhpd->current_key.len = val.len;
uhpd->original_key = val;
+ uhpd->hash = ngx_http_upstream_hash_crc32(uhpd->current_key.data, uhpd->current_key.len);
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "upstream_hash: hashed \"%V\" to %ui", &uhpd->current_key, uhpd->hash % uhpd->peers->number);
+ uhpd->try_i = 0;
+
+ /* In case this one is marked down */
+ ngx_http_upstream_hash_next_peer(uhpd, &r->upstream->peer.tries, r->connection->log);
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
- "upstream_hash: hashing \"%V\"", &val);
+ "upstream_hash: Starting with %ui", uhpd->hash % uhpd->peers->number);
- uhpd->hash = ngx_http_upstream_hash_crc32(val.data, val.len);
return NGX_OK;
}
@@ -207,8 +223,9 @@ ngx_http_upstream_get_hash_peer(ngx_peer_connection_t *pc, void *data)
peer = &uhpd->peers->peer[peer_index];
- ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
- "upstream_hash: chose peer %ui w/ hash %ui", peer_index, uhpd->hash);
+
+ ngx_log_debug3(NGX_LOG_DEBUG_HTTP, pc->log, 0,
+ "upstream_hash: chose peer %ui w/ hash %ui for tries %ui", peer_index, uhpd->hash, pc->tries);
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
@@ -225,29 +242,42 @@ ngx_http_upstream_free_hash_peer(ngx_peer_connection_t *pc, void *data,
ngx_http_upstream_hash_peer_data_t *uhpd = data;
ngx_uint_t current;
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"upstream_hash: free upstream hash peer try %ui", pc->tries);
if (state & (NGX_PEER_FAILED|NGX_PEER_NEXT)
- && --pc->tries)
- {
+ && pc->tries) {
current = uhpd->hash % uhpd->peers->number;
uhpd->tried[ngx_bitvector_index(current)] |= ngx_bitvector_bit(current);
-
- do {
- uhpd->current_key.len = ngx_sprintf(uhpd->current_key.data, "%d%V",
- ++uhpd->try_i, &uhpd->original_key) - uhpd->current_key.data;
- ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
- "upstream_hash: hashing \"%V\"", &uhpd->current_key);
- uhpd->hash += ngx_http_upstream_hash_crc32(uhpd->current_key.data,
- uhpd->current_key.len);
- current = uhpd->hash % uhpd->peers->number;
- } while ((uhpd->tried[ngx_bitvector_index(current)] & ngx_bitvector_bit(current))
- && --pc->tries);
+ ngx_http_upstream_hash_next_peer(uhpd, &pc->tries, pc->log);
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
+ "upstream_hash: Using %ui because %ui failed", uhpd->hash % uhpd->peers->number, current);
+ } else {
+ pc->tries = 0;
}
}
+static void ngx_http_upstream_hash_next_peer(ngx_http_upstream_hash_peer_data_t *uhpd,
+ ngx_uint_t *tries, ngx_log_t *log) {
+
+ ngx_uint_t current;
+ current = uhpd->hash % uhpd->peers->number;
+ // Loop while there is a try left, we're on one we haven't tried, and
+ // the current peer isn't marked down
+ while ((*tries)-- && (
+ (uhpd->tried[ngx_bitvector_index(current)] & ngx_bitvector_bit(current))
+ || uhpd->peers->peer[current].down)) {
+ uhpd->current_key.len = ngx_sprintf(uhpd->current_key.data, "%d%V",
+ ++uhpd->try_i, &uhpd->original_key) - uhpd->current_key.data;
+ uhpd->hash += ngx_http_upstream_hash_crc32(uhpd->current_key.data,
+ uhpd->current_key.len);
+ current = uhpd->hash % uhpd->peers->number;
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, log, 0,
+ "upstream_hash: hashed \"%V\" to %ui", &uhpd->current_key, current);
+ }
+}
+
/* bit-shift, bit-mask, and non-zero requirement are for libmemcache compatibility */
static ngx_uint_t
ngx_http_upstream_hash_crc32(u_char *keydata, size_t keylen)
@@ -286,7 +316,7 @@ ngx_http_upstream_hash(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
uscf->peer.init_upstream = ngx_http_upstream_init_hash;
- uscf->flags = NGX_HTTP_UPSTREAM_CREATE;
+ uscf->flags = NGX_HTTP_UPSTREAM_CREATE | NGX_HTTP_UPSTREAM_DOWN;
uscf->values = vars_values->elts;
uscf->lengths = vars_lengths->elts;

0 comments on commit fdaa270

Please sign in to comment.