diff --git a/.travis.yml b/.travis.yml index 6ae87e6..dd87c8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ addons: - liburi-perl - libwww-perl - perl + - valgrind cache: directories: @@ -28,9 +29,12 @@ cache: env: global: - - JOBS=2 - - OPENRESTY_PREFIX=/usr/local/openresty - - OPENRESTY_VER=1.11.2.3 + - JOBS=2 + - OPENRESTY_PREFIX=/usr/local/openresty + - OPENRESTY_VER=1.19.9.1 + jobs: + - TEST_NGINX_USE_VALGRIND=0 + - TEST_NGINX_USE_VALGRIND=1 install: - if [ ! -f download-cache/openresty-$OPENRESTY_VER.tar.gz ]; then @@ -38,15 +42,31 @@ install: fi - git clone https://github.com/openresty/test-nginx.git ../test-nginx +# install openresty-openssl111-dev +- sudo apt-get -y install --no-install-recommends wget gnupg ca-certificates +- wget -O - https://openresty.org/package/pubkey.gpg | sudo apt-key add - +- echo "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main" + | sudo tee /etc/apt/sources.list.d/openresty.list +- sudo apt-get update || echo 'apt-get update failed, but ignore it' +- sudo apt-get -y install --no-install-recommends openresty-openssl111 openresty-openssl111-dev + script: +- if [ TEST_NGINX_USE_VALGRIND = 1 ]; then export luajit_xcflags='-DLUAJIT_USE_VALGRIND -DLUAJIT_USE_SYSMALLOC'; fi - tar xzf download-cache/openresty-$OPENRESTY_VER.tar.gz && cd openresty-$OPENRESTY_VER -- ./configure --prefix=$OPENRESTY_PREFIX -j$JOBS - > build.log 2>&1 || (cat build.log && exit 1) +- ./configure --prefix=$OPENRESTY_PREFIX + --with-cc-opt="-I/usr/local/openresty/openssl111/include" + --with-ld-opt="-L/usr/local/openresty/openssl111/lib -Wl,-rpath,/usr/local/openresty/openssl111/lib" + --with-luajit-xcflags="$luajit_xcflags" + -j$JOBS + > build.log 2>&1 || (cat build.log && exit 1) - make -j$JOBS > build.log 2>&1 || (cat build.log && exit 1) - sudo make install > build.log 2>&1 || (cat build.log && exit 1) - cd .. - export PATH=$OPENRESTY_PREFIX/nginx/sbin:$PATH -- make test jobs=$JOBS +- make test jobs=$JOBS > build.log 2>&1 || + (cat build.log && exit 1) +- cat build.log +- if [ `grep -c '== Invalid' build.log` -gt 0 ]; then echo 'valgrind complaining' && exit 1; fi diff --git a/Makefile b/Makefile index 90fc0ad..533df89 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,11 @@ LDFLAGS := -shared # on Mac OS X, one should set instead: # LDFLAGS := -bundle -undefined dynamic_lookup +ifeq ($(shell uname),Darwin) + LDFLAGS := -bundle -undefined dynamic_lookup + C_SO_NAME := librestychash.dylib +endif + MY_CFLAGS := $(CFLAGS) -DBUILDING_SO MY_LDFLAGS := $(LDFLAGS) -fvisibility=hidden @@ -39,7 +44,9 @@ clean:; rm -f *.o *.so a.out *.d install: $(INSTALL) -d $(DESTDIR)$(LUA_LIB_DIR)/resty + $(INSTALL) -d $(DESTDIR)$(LUA_LIB_DIR)/resty/balancer $(INSTALL) lib/resty/*.lua $(DESTDIR)$(LUA_LIB_DIR)/resty + $(INSTALL) lib/resty/balancer/*.lua $(DESTDIR)$(LUA_LIB_DIR)/resty/balancer $(INSTALL) $(C_SO_NAME) $(DESTDIR)$(LUA_LIB_DIR)/ test : all diff --git a/README.markdown b/README.markdown index 2317a52..b66c19c 100644 --- a/README.markdown +++ b/README.markdown @@ -101,6 +101,7 @@ Synopsis local rr_up = package.loaded.my_rr_up + -- Note that Round Robin picks the first server randomly local server = rr_up:find() assert(b.set_current_peer(server)) @@ -137,6 +138,7 @@ The `id` should be `table.concat({host, string.char(0), port})` like the nginx c when we need to keep consistency with nginx chash. The `id` can be any string value when we do not need to keep consistency with nginx chash. +The `weight` should be a non negative integer. ```lua local nodes = { diff --git a/chash.c b/chash.c index 1d13dd3..5c3148b 100644 --- a/chash.c +++ b/chash.c @@ -1,6 +1,8 @@ #include #include #include +#include + #include "chash.h" @@ -164,7 +166,9 @@ chash_point_sort(chash_point_t arr[], uint32_t n) for (i = 0; i < n; i++) { node = &arr[i]; - index = node->hash / step; // can not bigger than m + index = node->hash / step; + + assert(index < m); // index must less than m for (end = index; end >= 0; end--) { if (points[end].id == 0) { @@ -188,7 +192,10 @@ chash_point_sort(chash_point_t arr[], uint32_t n) /* left shift after end when node->hash is bigger than them */ /* only end == index can match this */ - while (points[end + 1].id != 0 && points[end + 1].hash < node->hash) { + while (end + 1 < m + && points[end + 1].id != 0 + && points[end + 1].hash < node->hash) + { points[end].hash = points[end + 1].hash; points[end].id = points[end + 1].id; end += 1; @@ -223,6 +230,8 @@ chash_point_sort(chash_point_t arr[], uint32_t n) } insert: + assert(end < m && end >= 0); + points[end].id = node->id; points[end].hash = node->hash; } diff --git a/lib/resty/balancer/utils.lua b/lib/resty/balancer/utils.lua new file mode 100644 index 0000000..dd749dc --- /dev/null +++ b/lib/resty/balancer/utils.lua @@ -0,0 +1,47 @@ +local _M = {} + +_M.name = "balancer-utils" +_M.version = "0.03" + +local new_tab +do + local ok + ok, new_tab = pcall(require, "table.new") + if not ok or type(new_tab) ~= "function" then + new_tab = function (narr, nrec) return {} end + end +end +_M.new_tab = new_tab + + +local nkeys, tab_nkeys +do + local ok + ok, nkeys = pcall(require, "table.nkeys") + if not ok or type(nkeys) ~= "function" then + nkeys = function(tab) + local count = 0 + for _, _ in pairs(tab) do + count = count + 1 + end + return count + end + + else + tab_nkeys = nkeys + end +end +_M.nkeys = nkeys + + +function _M.copy(nodes) + local newnodes = new_tab(0, tab_nkeys and tab_nkeys(nodes) or 4) + for id, weight in pairs(nodes) do + newnodes[id] = tonumber(weight) + end + + return newnodes +end + + +return _M diff --git a/lib/resty/chash.lua b/lib/resty/chash.lua index ed8e25d..90e73bc 100644 --- a/lib/resty/chash.lua +++ b/lib/resty/chash.lua @@ -5,6 +5,10 @@ local bit = require "bit" local ffi = require 'ffi' +local utils = require "resty.balancer.utils" + +local new_tab = utils.new_tab +local copy = utils.copy local ffi_new = ffi.new local C = ffi.C @@ -38,13 +42,6 @@ void chash_point_delete(chash_point_t *old_points, uint32_t old_length, uint32_t id); ]] - -local ok, new_tab = pcall(require, "table.new") -if not ok or type(new_tab) ~= "function" then - new_tab = function (narr, nrec) return {} end -end - - -- -- Find shared object file package.cpath, obviating the need of setting -- LD_LIBRARY_PATH @@ -58,6 +55,12 @@ local function load_shared_lib(so_name) local cpath = package.cpath + local postfix = ".so" + if ffi.os == "OSX" then + postfix = ".dylib" + end + so_name = so_name .. postfix + for k, _ in string_gmatch(cpath, "[^;]+") do local fpath = string_match(k, "(.*/)") fpath = fpath .. so_name @@ -77,9 +80,9 @@ local _M = {} local mt = { __index = _M } -local clib = load_shared_lib("librestychash.so") +local clib = load_shared_lib("librestychash") if not clib then - error("can not load librestychash.so") + error("can not load librestychash") end local CONSISTENT_POINTS = 160 -- points per server @@ -95,10 +98,7 @@ local function _precompute(nodes) total_weight = total_weight + weight end - local newnodes = new_tab(0, n) - for id, weight in pairs(nodes) do - newnodes[id] = weight - end + local newnodes = copy(nodes) local ids = new_tab(n, 0) local npoints = total_weight * CONSISTENT_POINTS diff --git a/lib/resty/roundrobin.lua b/lib/resty/roundrobin.lua index 5f9f19d..97743b5 100644 --- a/lib/resty/roundrobin.lua +++ b/lib/resty/roundrobin.lua @@ -3,22 +3,17 @@ local pairs = pairs local next = next local tonumber = tonumber local setmetatable = setmetatable +local math_random = math.random +local utils = require "resty.balancer.utils" + +local copy = utils.copy +local nkeys = utils.nkeys +local new_tab = utils.new_tab local _M = {} local mt = { __index = _M } - -local function copy(nodes) - local newnodes = {} - for id, weight in pairs(nodes) do - newnodes[id] = weight - end - - return newnodes -end - - local _gcd _gcd = function (a, b) if b == 0 then @@ -46,10 +41,24 @@ local function get_gcd(nodes) return only_key, gcd, max_weight end +local function get_random_node_id(nodes) + local count = nkeys(nodes) + + local id = nil + local random_index = math_random(count) + + for _ = 1, random_index do + id = next(nodes, id) + end + + return id +end + function _M.new(_, nodes) local newnodes = copy(nodes) local only_key, gcd, max_weight = get_gcd(newnodes) + local last_id = get_random_node_id(nodes) local self = { nodes = newnodes, -- it's safer to copy one @@ -57,7 +66,7 @@ function _M.new(_, nodes) max_weight = max_weight, gcd = gcd, cw = max_weight, - last_id = nil, + last_id = last_id, } return setmetatable(self, mt) end @@ -68,7 +77,7 @@ function _M.reinit(self, nodes) self.only_key, self.gcd, self.max_weight = get_gcd(newnodes) self.nodes = newnodes - self.last_id = nil + self.last_id = get_random_node_id(nodes) self.cw = self.max_weight end diff --git a/lua-resty-balancer-0.04-0.rockspec b/lua-resty-balancer-0.04-0.rockspec new file mode 100644 index 0000000..ca75831 --- /dev/null +++ b/lua-resty-balancer-0.04-0.rockspec @@ -0,0 +1,24 @@ +package = "lua-resty-balancer" +version = "0.04-0" +source = { + url = "git://github.com/openresty/lua-resty-balancer", + tag = "v0.04", +} + +description = { + summary = "A generic consistent hash implementation for OpenResty", + homepage = "https://github.com/openresty/lua-resty-balancer", + license = "Apache License 2.0", + maintainer = "Yichun Zhang (agentzh) ", +} + +build = { + type = "builtin", + modules = { + ["librestychash"] = {"chash.c"}, + + ["resty.chash"] = "lib/resty/chash.lua", + ["resty.roundrobin"] = "lib/resty/roundrobin.lua", + ["resty.balancer.utils"] = "lib/resty/balancer/utils.lua", + } +} diff --git a/t/chash.t b/t/chash.t index 1ba5445..10ed6a9 100644 --- a/t/chash.t +++ b/t/chash.t @@ -41,7 +41,6 @@ __DATA__ local res = {} for i = 1, 100 * 1000 do local id = chash:find(i) - if res[id] then res[id] = res[id] + 1 else @@ -49,8 +48,9 @@ __DATA__ end end - for id, num in pairs(res) do - ngx.say(id, ": ", num) + for i=1, 3 do + local id = "server"..i + ngx.say(id..": ", res[id]) end ngx.say("points number: ", chash.npoints) @@ -59,8 +59,8 @@ __DATA__ --- request GET /t --- response_body -server2: 14743 server1: 77075 +server2: 14743 server3: 8182 points number: 2080 --- no_error_log @@ -304,9 +304,18 @@ diff: 9745 local chash = resty_chash:new(servers) + local success = true + local count = 0 + for id, weight in pairs(chash.nodes) do - ngx.say(id, ": ", weight) + count = count + 1 + if servers[id] ~= weight then + success = false + end end + ngx.say("count: ", count) + ngx.say("success: ",success) + ngx.say("points number: ", chash.npoints) ngx.say("size: ", chash.size) @@ -318,9 +327,16 @@ diff: 9745 } chash:reinit(new_servers) + count = 0 for id, weight in pairs(chash.nodes) do - ngx.say(id, ": ", weight) + count = count + 1 + if new_servers[id] ~= weight then + success = false + end end + ngx.say("count: ", count) + ngx.say("success: ",success) + ngx.say("points number: ", chash.npoints) ngx.say("size: ", chash.size) } @@ -328,16 +344,60 @@ diff: 9745 --- request GET /t --- response_body -server1: 10 -server2: 2 -server3: 1 +count: 3 +success: true points number: 2080 size: 2080 reinit -server4: 1 -server5: 2 +count: 2 +success: true points number: 480 size: 480 --- no_error_log [error] --- timeout: 30 + + + +=== TEST 6: random key fuzzer +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + math.randomseed(ngx.now()) + + local ffi = require "ffi" + local resty_chash = require "resty.chash" + + local function random_string() + local len = math.random(10, 100) + local buf = ffi.new("char [?]", len) + for i = 0, len - 1 do + buf[i] = math.random(0, 255) + end + + return ffi.string(buf, len) + end + + for i = 1, 30 do + local servers = {} + + local len = math.random(1, 100) + for j = 1, len do + local key = random_string() + servers[key] = math.random(1, 100) + end + + local chash = resty_chash:new(servers) + end + + ngx.say("done") + } + } +--- request +GET /t +--- response_body +done +--- no_error_log +[error] +--- timeout: 30 diff --git a/t/roundrobin.t b/t/roundrobin.t index 21dba86..74d4a8a 100644 --- a/t/roundrobin.t +++ b/t/roundrobin.t @@ -28,6 +28,8 @@ __DATA__ --- config location /t { content_by_lua_block { + math.randomseed(75098) + local roundrobin = require "resty.roundrobin" local servers = { @@ -42,29 +44,19 @@ __DATA__ for i = 1, 14 do local id = rr:find() - - ngx.say("id: ", id) + if type(id) ~= "string" or not servers[id] then + return ngx.say("fail") + end end + + ngx.say("success") } } --- request GET /t --- response_body gcd: 2 -id: server1 -id: server1 -id: server1 -id: server2 -id: server1 -id: server2 -id: server3 -id: server1 -id: server1 -id: server1 -id: server2 -id: server1 -id: server2 -id: server3 +success --- no_error_log [error] @@ -75,12 +67,14 @@ id: server3 --- config location /t { content_by_lua_block { + math.randomseed(75098) + local roundrobin = require "resty.roundrobin" local servers = { ["server1"] = 6, - ["server2"] = 4, - ["server3"] = 2, + ["server2"] = 3, + ["server3"] = 1, } local rr = roundrobin:new(servers) @@ -96,16 +90,125 @@ id: server3 end end + local keys = {} for id, num in pairs(res) do - ngx.say(id, ": ", num) + keys[#keys + 1] = id end + + if #keys ~= 3 then + ngx.exit(400) + end + + ngx.say("server1: ", res['server1']) + ngx.say("server2: ", res['server2']) + ngx.say("server3: ", res['server3']) + } + } +--- request +GET /t +--- response_body +server1: 60000 +server2: 30000 +server3: 10000 +--- no_error_log +[error] + + + +=== TEST 3: random start +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + math.randomseed(9975098) + + local roundrobin = require "resty.roundrobin" + + local servers = { + ["server1"] = 1, + ["server2"] = 1, + ["server3"] = 1, + ["server4"] = 1, + } + + local rr = roundrobin:new(servers, true) + local id = rr:find() + + local rr2 = roundrobin:new(servers, true) + local id2 = rr2:find() + ngx.log(ngx.INFO, "id: ", id, " id2: ", id2) + ngx.say(id == id2) + } + } +--- request +GET /t +--- response_body +false +--- no_error_log +[error] + + + +=== TEST 4: weight is "0" +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + math.randomseed(9975098) + + local roundrobin = require "resty.roundrobin" + + local servers = { + ["server1"] = "0", + ["server2"] = "1", + ["server3"] = "0", + ["server4"] = "0", + } + + local rr = roundrobin:new(servers, true) + local id = rr:find() + + ngx.say("id: ", id) + } + } +--- request +GET /t +--- response_body +id: server2 +--- no_error_log +[error] + + + +=== TEST 5: all weights are 0, behavior like weights are 1. +It's not recommends to use 0, this test just make sure it won't be worse, like crash. +--- http_config eval: $::HttpConfig +--- config + location /t { + content_by_lua_block { + math.randomseed(9975098) + + local roundrobin = require "resty.roundrobin" + + local servers = { + ["server1"] = 0, + ["server2"] = 0, + ["server3"] = 0, + ["server4"] = 0, + } + + local rr = roundrobin:new(servers, true) + + for i = 1, 4 do + local id = rr:find() + end + + ngx.say("ok") } } --- request GET /t --- response_body -server1: 50001 -server3: 16666 -server2: 33333 +ok --- no_error_log [error]