feat(router) allow to route TLS request without terminating it

kong/constants.lua

@@ -56,6 +56,7 @@ local protocols_with_subsystem = {
   tcp = "stream",
   tls = "stream",
   udp = "stream",
+  tls_passthrough = "stream",
   grpc = "http",
   grpcs = "http",
 }

kong/db/schema/entities/routes.lua

@@ -19,7 +19,8 @@ return {
                          elements = typedefs.protocol,
                          mutually_exclusive_subsets = {
                            { "http", "https" },
-                           { "tcp", "tls", "udp", },
+                           { "tcp", "tls", "udp" },
+                           { "tls_passthrough" },
                            { "grpc", "grpcs" },
                          },
                          default = { "http", "https" }, -- TODO: different default depending on service's scheme
@@ -57,7 +58,7 @@ return {
 
   entity_checks = {
     { conditional = { if_field = "protocols",
-                      if_match = { elements = { type = "string", not_one_of = { "grpcs", "https", "tls" }}},
+                      if_match = { elements = { type = "string", not_one_of = { "grpcs", "https", "tls", "tls_passthrough" }}},
                       then_field = "snis",
                       then_match = { len_eq = 0 },
                       then_err = "'snis' can only be set when 'protocols' is 'grpcs', 'https' or 'tls'",

kong/db/schema/entities/routes_subschemas.lua

@@ -24,16 +24,21 @@ local stream_subschema = {
   name = "tcp",
 
   fields = {
-    { methods = typedefs.no_methods { err = "cannot set 'methods' when 'protocols' is 'tcp', 'tls' or 'udp'" } },
-    { hosts = typedefs.no_hosts { err = "cannot set 'hosts' when 'protocols' is 'tcp', 'tls' or 'udp'" } },
-    { paths = typedefs.no_paths { err = "cannot set 'paths' when 'protocols' is 'tcp', 'tls' or 'udp'" } },
-    { headers = typedefs.no_headers { err = "cannot set 'headers' when 'protocols' is 'tcp', 'tls' or 'udp'" } },
+    { methods = typedefs.no_methods { err = "cannot set 'methods' when 'protocols' is 'tcp', 'tls', 'tls_passthrough' or 'udp'" } },
+    { hosts = typedefs.no_hosts { err = "cannot set 'hosts' when 'protocols' is 'tcp', 'tls', 'tls_passthrough' or 'udp'" } },
+    { paths = typedefs.no_paths { err = "cannot set 'paths' when 'protocols' is 'tcp', 'tls', 'tls_passthrough' or 'udp'" } },
+    { headers = typedefs.no_headers { err = "cannot set 'headers' when 'protocols' is 'tcp', 'tls', 'tls_passthrough' or 'udp'" } },
   },
   entity_checks = {
     { conditional_at_least_one_of = { if_field = "protocols",
                                       if_match = { elements = { type = "string", one_of = { "tcp", "tls", "udp", } } },
                                       then_at_least_one_of = { "sources", "destinations", "snis" },
-                                      then_err = "must set one of %s when 'protocols' is 'tcp', 'tls' or 'udp'",
+                                      then_err = "must set one of %s when 'protocols' is 'tcp', 'tls', 'tls_passthrough' or 'udp'",
+                                    }},
+    {conditional_at_least_one_of = { if_field = "protocols",
+                                      if_match = { elements = { type = "string", one_of = { "tls_passthrough" } } },
+                                      then_at_least_one_of = { "snis" },
+                                      then_err = "must set snis when 'protocols' is 'tls_passthrough'",
                                     }},
   },
 }
@@ -67,6 +72,7 @@ return {
   tcp = stream_subschema,
   tls = stream_subschema,
   udp = stream_subschema,
+  tls_passthrough = stream_subschema,
   grpc = grpc_subschema,
   grpcs = grpc_subschema,
 }

kong/init.lua

@@ -742,6 +742,12 @@ function Kong.preread()
 
   runloop.preread.before(ctx)
 
+  -- if proxying to a second layer TLS terminator is required
+  -- abort further execution and return back to Nginx
+  if ctx.stream_proxy_preread_terminate then
+    return
+  end
+
   local plugins_iterator = runloop.get_updated_plugins_iterator()
   execute_plugins_iterator(plugins_iterator, "preread", ctx)
 

kong/pdk/client.lua

@@ -25,6 +25,8 @@ local AUTH_AND_LATER = phase_checker.new(PHASES.access,
                                          PHASES.log)
 local TABLE_OR_NIL = { ["table"] = true, ["nil"] = true }
 
+local stream_subsystem = ngx.config.subsystem == "stream"
+
 
 local function new(self)
   local _CLIENT = {}
@@ -48,6 +50,14 @@ local function new(self)
   function _CLIENT.get_ip()
     check_not_phase(PHASES.init_worker)
 
+    -- when proxying TLS request in second layer or doing TLS passthrough
+    -- realip_remote_addr is always the previous layer of nginx thus always unix:
+    local tls_passthrough_block = ngx.var.kong_tls_passthrough_block
+    if stream_subsystem and
+        ((tls_passthrough_block and #tls_passthrough_block > 0) or ngx.var.ssl_protocol) then
+      return ngx.var.remote_addr
+    end
+
     return ngx.var.realip_remote_addr or ngx.var.remote_addr
   end
 
@@ -99,6 +109,14 @@ local function new(self)
   function _CLIENT.get_port()
     check_not_phase(PHASES.init_worker)
 
+    -- when proxying TLS request in second layer or doing TLS passthrough
+    -- realip_remote_addr is always the previous layer of nginx thus always unix:
+    local tls_passthrough_block = ngx.var.kong_tls_passthrough_block
+    if stream_subsystem and
+        ((tls_passthrough_block and #tls_passthrough_block > 0) or ngx.var.ssl_protocol) then
+      return tonumber(ngx.var.remote_port)
+    end
+
     return tonumber(ngx.var.realip_remote_port or ngx.var.remote_port)
   end
 

kong/router.lua

@@ -1897,6 +1897,10 @@ function _M.new(routes)
                     or tonumber(var.server_port, 10)
       -- error value for non-TLS connections ignored intentionally
       local sni, _ = server_name()
+      -- fallback to preread SNI if current connection doesn't terminate TLS
+      if not sni then
+        sni = var.ssl_preread_server_name
+      end
 
       local scheme
       if var.protocol == "UDP" then
@@ -1906,6 +1910,14 @@ function _M.new(routes)
         scheme = sni and "tls" or "tcp"
       end
 
+      -- when proxying TLS request in second layer or doing TLS passthrough
+      -- rewrite the dst_ip,port back to what specified in proxy_protocol
+      local tls_passthrough_block = var.kong_tls_passthrough_block
+      if (tls_passthrough_block and #tls_passthrough_block > 0) or var.ssl_protocol then
+        dst_ip = var.proxy_protocol_server_addr
+        dst_port = tonumber(var.proxy_protocol_server_port)
+      end
+
       return find_route(nil, nil, nil, scheme,
                         src_ip, src_port,
                         dst_ip, dst_port,

kong/runloop/handler.lua

@@ -1113,9 +1113,32 @@ return {
         return exit(500)
       end
 
+      local route = match_t.route
+      -- if matched route doesn't do tls_passthrough and we are in the preread server block
+      -- this request should be TLS terminated; return immediately and not run further steps
+      -- (even bypassing the balancer)
+      local tls_preread_block = var.kong_tls_preread_block
+      if tls_preread_block and #tls_preread_block > 0 then
+        local protocols = route.protocols
+        if protocols and protocols.tls then
+          log(DEBUG, "TLS termination required, return to second layer proxying")
+          var.kong_tls_preread_block_upstream = "unix:" .. kong.configuration.prefix .. "/stream_tls_terminate.sock"
+
+        elseif protocols and protocols.tls_passthrough then
+          var.kong_tls_preread_block_upstream = "unix:" .. kong.configuration.prefix .. "/stream_tls_passthrough.sock"
+
+        else
+          log(ERR, "unexpected protocols in matched Route")
+          return exit(500)
+        end
+
+        ctx.stream_proxy_preread_terminate = true
+        return
+      end
+
+
       ctx.workspace = match_t.route and match_t.route.ws_id
 
-      local route = match_t.route
       local service = match_t.service
       local upstream_url_t = match_t.upstream_url_t
 

kong/templates/nginx_kong_stream.lua

@@ -86,17 +86,25 @@ upstream kong_upstream {
 }
 
 > if #stream_listeners > 0 then
+# non-SSL listeners, and the SSL terminator
 server {
 > for _, entry in ipairs(stream_listeners) do
+> if not entry.ssl then
     listen $(entry.listener);
 > end
+> end
+
+> if stream_proxy_ssl_enabled then
+    listen unix:${{PREFIX}}/stream_tls_terminate.sock ssl proxy_protocol;
+> end
 
     access_log ${{PROXY_STREAM_ACCESS_LOG}};
     error_log ${{PROXY_STREAM_ERROR_LOG}} ${{LOG_LEVEL}};
 
 > for _, ip in ipairs(trusted_ips) do
     set_real_ip_from $(ip);
 > end
+    set_real_ip_from unix:;
 
     # injected nginx_sproxy_* directives
 > for _, el in ipairs(nginx_sproxy_directives) do
@@ -131,6 +139,69 @@ server {
     }
 }
 
+> if stream_proxy_ssl_enabled then
+# SSL listeners, but only preread the handshake here
+server {
+> for _, entry in ipairs(stream_listeners) do
+> if entry.ssl then
+    listen $(entry.listener:gsub(" ssl", ""));
+> end
+> end
+
+    access_log ${{PROXY_STREAM_ACCESS_LOG}};
+    error_log ${{PROXY_STREAM_ERROR_LOG}} ${{LOG_LEVEL}};
+
+> for _, ip in ipairs(trusted_ips) do
+    set_real_ip_from $(ip);
+> end
+
+    # injected nginx_sproxy_* directives
+> for _, el in ipairs(nginx_sproxy_directives) do
+    $(el.name) $(el.value);
+> end
+
+    preread_by_lua_block {
+        Kong.preread()
+    }
+
+    ssl_preread on;
+
+    proxy_protocol on;
+
+    set $kong_tls_preread_block 1;
+    set $kong_tls_preread_block_upstream '';
+    proxy_pass $kong_tls_preread_block_upstream;
+}
+
+server {
+    listen unix:${{PREFIX}}/stream_tls_passthrough.sock proxy_protocol;
+
+    access_log ${{PROXY_STREAM_ACCESS_LOG}};
+    error_log ${{PROXY_STREAM_ERROR_LOG}} ${{LOG_LEVEL}};
+
+    set_real_ip_from unix:;
+
+    # injected nginx_sproxy_* directives
+> for _, el in ipairs(nginx_sproxy_directives) do
+    $(el.name) $(el.value);
+> end
+
+    preread_by_lua_block {
+        Kong.preread()
+    }
+
+    ssl_preread on;
+
+    set $kong_tls_passthrough_block 1;
+
+    proxy_pass kong_upstream;
+
+    log_by_lua_block {
+        Kong.log()
+    }
+}
+> end -- stream_proxy_ssl_enabled
+
 > if database == "off" then
 server {
     listen unix:${{PREFIX}}/stream_config.sock;

spec/01-unit/01-db/01-schema/06-routes_spec.lua

@@ -794,7 +794,7 @@ describe("routes schema", function()
         local ok, errs = Routes:validate(route)
         assert.falsy(ok)
         assert.same({
-          paths = "cannot set 'paths' when 'protocols' is 'tcp', 'tls' or 'udp'",
+          paths = "cannot set 'paths' when 'protocols' is 'tcp', 'tls', 'tls_passthrough' or 'udp'",
         }, errs)
       end
     end)
@@ -811,7 +811,7 @@ describe("routes schema", function()
         local ok, errs = Routes:validate(route)
         assert.falsy(ok)
         assert.same({
-          methods = "cannot set 'methods' when 'protocols' is 'tcp', 'tls' or 'udp'",
+          methods = "cannot set 'methods' when 'protocols' is 'tcp', 'tls', 'tls_passthrough' or 'udp'",
         }, errs)
       end
     end)
@@ -1038,7 +1038,7 @@ describe("routes schema", function()
         assert.falsy(ok)
         assert.same({
           ["@entity"] = {
-            "must set one of 'sources', 'destinations', 'snis' when 'protocols' is 'tcp', 'tls' or 'udp'"
+            "must set one of 'sources', 'destinations', 'snis' when 'protocols' is 'tcp', 'tls', 'tls_passthrough' or 'udp'"
           }
         }, errs)
       end

spec/01-unit/01-db/01-schema/11-declarative_config/01-validate_spec.lua

@@ -190,7 +190,7 @@ describe("declarative config: validate", function()
               ["host"] = "expected a string",
               ["path"] = "must not have empty segments",
               ["port"] = "value should be between 0 and 65535",
-              ["protocol"] = "expected one of: grpc, grpcs, http, https, tcp, tls, udp",
+              ["protocol"] = "expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp",
               ["retries"] = "value should be between 0 and 32767",
             }
           }

spec/02-integration/02-cmd/02-start_stop_spec.lua

@@ -532,7 +532,7 @@ describe("kong start/stop #" .. strategy, function()
         })
 
         assert.falsy(ok)
-        assert.matches("in 'protocol': expected one of: grpc, grpcs, http, https, tcp, tls, udp", err, nil, true)
+        assert.matches("in 'protocol': expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp", err, nil, true)
         assert.matches("in 'name': invalid value '@gobo': the only accepted ascii characters are alphanumerics or ., -, _, and ~", err, nil, true)
         assert.matches("in entry 2 of 'hosts': invalid hostname: \\\\99", err, nil, true)
       end)

spec/02-integration/04-admin_api/09-routes_routes_spec.lua

@@ -364,9 +364,9 @@ for _, strategy in helpers.each_strategy() do
                 code    = Errors.codes.SCHEMA_VIOLATION,
                 name    = "schema violation",
                 message = "schema violation " ..
-                          "(protocols.1: expected one of: grpc, grpcs, http, https, tcp, tls, udp)",
+                          "(protocols.1: expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp)",
                 fields = {
-                  protocols = { "expected one of: grpc, grpcs, http, https, tcp, tls, udp" },
+                  protocols = { "expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp" },
                 }
               }, cjson.decode(body))
 
@@ -384,12 +384,12 @@ for _, strategy in helpers.each_strategy() do
                 code    = Errors.codes.SCHEMA_VIOLATION,
                 name    = "schema violation",
                 message = "2 schema violations " ..
-                  "(protocols.1: expected one of: grpc, grpcs, http, https, tcp, tls, udp; " ..
-                  "service.protocol: expected one of: grpc, grpcs, http, https, tcp, tls, udp)",
+                  "(protocols.1: expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp; " ..
+                  "service.protocol: expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp)",
                 fields = {
-                  protocols = { "expected one of: grpc, grpcs, http, https, tcp, tls, udp" },
+                  protocols = { "expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp" },
                   service = {
-                    protocol = "expected one of: grpc, grpcs, http, https, tcp, tls, udp"
+                    protocol = "expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp"
                   }
                 }
               }, cjson.decode(body))
@@ -928,9 +928,9 @@ for _, strategy in helpers.each_strategy() do
                   code    = Errors.codes.SCHEMA_VIOLATION,
                   name    = "schema violation",
                   message = "schema violation " ..
-                    "(protocols.1: expected one of: grpc, grpcs, http, https, tcp, tls, udp)",
+                    "(protocols.1: expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp)",
                   fields  = {
-                    protocols = { "expected one of: grpc, grpcs, http, https, tcp, tls, udp" },
+                    protocols = { "expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp" },
                   }
                 }, cjson.decode(body))
 

spec/02-integration/04-admin_api/10-services_routes_spec.lua

@@ -822,7 +822,7 @@ for _, strategy in helpers.each_strategy() do
               })
             body = assert.res_status(400, res)
             json = cjson.decode(body)
-            assert.same({ protocol = "expected one of: grpc, grpcs, http, https, tcp, tls, udp" }, json.fields)
+            assert.same({ protocol = "expected one of: grpc, grpcs, http, https, tcp, tls, tls_passthrough, udp" }, json.fields)
           end
         end)