fix(consul): retry connecting after a delay (#4979)

apache · Sep 6, 2021 · e49b41d · e49b41d
1 parent 0ba89d5
commit e49b41d
Show file tree

Hide file tree

Showing 2 changed files with 60 additions and 2 deletions.
diff --git a/apisix/discovery/consul_kv.lua b/apisix/discovery/consul_kv.lua
@@ -17,6 +17,7 @@
 local require            = require
 local local_conf         = require("apisix.core.config_local").local_conf()
 local core               = require("apisix.core")
+local core_sleep         = require("apisix.core.utils").sleep
 local resty_consul       = require('resty.consul')
 local cjson              = require('cjson')
 local http               = require('resty.http')
@@ -307,7 +308,7 @@ local function show_dump_file()
 end
 
 
-function _M.connect(premature, consul_server)
+function _M.connect(premature, consul_server, retry_delay)
     if premature then
         return
     end
@@ -331,6 +332,15 @@ function _M.connect(premature, consul_server)
             ", got result: ", json_delay_encode(result, true),
             ", with error: ", error_info)
 
+        if not retry_delay then
+            retry_delay = 1
+        else
+            retry_delay = retry_delay * 4
+        end
+
+        log.warn("retry connecting consul after ", retry_delay, " seconds")
+        core_sleep(retry_delay)
+
         goto ERR
     end
 
@@ -370,7 +380,7 @@ function _M.connect(premature, consul_server)
     :: ERR ::
     local keepalive = consul_server.keepalive
     if keepalive then
-        local ok, err = ngx_timer_at(0, _M.connect, consul_server)
+        local ok, err = ngx_timer_at(0, _M.connect, consul_server, retry_delay)
         if not ok then
             log.error("create ngx_timer_at got error: ", err)
             return

diff --git a/t/discovery/consul_kv.t b/t/discovery/consul_kv.t
@@ -514,3 +514,51 @@ location /sleep {
     qr/ok\n/,
     qr/server 1\n/
 ]
+
+
+
+=== TEST 12: retry when Consul can't be reached (long connect type)
+--- yaml_config
+apisix:
+  node_listen: 1984
+  config_center: yaml
+  enable_admin: false
+
+discovery:
+  consul_kv:
+    servers:
+      - "http://127.0.0.1:8501"
+    keepalive: true
+    fetch_interval: 3
+    default_service:
+      host: "127.0.0.1"
+      port: 20999
+#END
+--- apisix_yaml
+routes:
+  -
+    uri: /*
+    upstream:
+      service_name: http://127.0.0.1:8501/v1/kv/upstreams/webpages/
+      discovery_type: consul_kv
+      type: roundrobin
+#END
+--- timeout: 4
+--- config
+location /sleep {
+    content_by_lua_block {
+        local args = ngx.req.get_uri_args()
+        local sec = args.sec or "2"
+        ngx.sleep(tonumber(sec))
+        ngx.say("ok")
+    }
+}
+--- request
+GET /sleep?sec=3
+--- response_body
+ok
+--- grep_error_log eval
+qr/retry connecting consul after \d seconds/
+--- grep_error_log_out
+retry connecting consul after 1 seconds
+retry connecting consul after 4 seconds