Merge branch 'master' into feat/ai_prompt_decorator_plugin

changelog/unreleased/kong/default_status_port.yml

@@ -0,0 +1,3 @@
+message: Enable `status_listen` on `127.0.0.1:8007` by default
+type: feature
+scope: Admin API

changelog/unreleased/kong/standardize-redis-conifguration-acme.yml

@@ -0,0 +1,3 @@
+message: "**ACME**: Standardize redis configuration across plugins. The redis configuration right now follows common schema that is shared across other plugins."
+type: deprecation
+scope: Plugin

changelog/unreleased/kong/support_net_src_dst_field_in_expression.yml

@@ -0,0 +1,4 @@
+message: |
+  `net.src.*` and `net.dst.*` match fields are now accessible in HTTP routes defined using expressions.
+type: feature
+scope: Core

kong-3.6.0-0.rockspec

@@ -181,6 +181,7 @@ build = {
     ["kong.tools.ip"] = "kong/tools/ip.lua",
     ["kong.tools.http"] = "kong/tools/http.lua",
     ["kong.tools.cjson"] = "kong/tools/cjson.lua",
+    ["kong.tools.redis.schema"] = "kong/tools/redis/schema.lua",
 
     ["kong.runloop.handler"] = "kong/runloop/handler.lua",
     ["kong.runloop.events"] = "kong/runloop/events.lua",
@@ -478,14 +479,18 @@ build = {
 
     ["kong.plugins.acme.api"] = "kong/plugins/acme/api.lua",
     ["kong.plugins.acme.client"] = "kong/plugins/acme/client.lua",
+    ["kong.plugins.acme.clustering.compat.redis_translation"] = "kong/plugins/acme/clustering/compat/redis_translation.lua",
     ["kong.plugins.acme.daos"] = "kong/plugins/acme/daos.lua",
     ["kong.plugins.acme.handler"] = "kong/plugins/acme/handler.lua",
     ["kong.plugins.acme.migrations.000_base_acme"] = "kong/plugins/acme/migrations/000_base_acme.lua",
     ["kong.plugins.acme.migrations.001_280_to_300"] = "kong/plugins/acme/migrations/001_280_to_300.lua",
     ["kong.plugins.acme.migrations.002_320_to_330"] = "kong/plugins/acme/migrations/002_320_to_330.lua",
+    ["kong.plugins.acme.migrations.003_350_to_360"] = "kong/plugins/acme/migrations/003_350_to_360.lua",
     ["kong.plugins.acme.migrations"] = "kong/plugins/acme/migrations/init.lua",
     ["kong.plugins.acme.schema"] = "kong/plugins/acme/schema.lua",
     ["kong.plugins.acme.storage.kong"] = "kong/plugins/acme/storage/kong.lua",
+    ["kong.plugins.acme.storage.config_adapters"] = "kong/plugins/acme/storage/config_adapters/init.lua",
+    ["kong.plugins.acme.storage.config_adapters.redis"] = "kong/plugins/acme/storage/config_adapters/redis.lua",
     ["kong.plugins.acme.reserved_words"] = "kong/plugins/acme/reserved_words.lua",
 
     ["kong.plugins.prometheus.api"] = "kong/plugins/prometheus/api.lua",

kong.conf.default

@@ -680,7 +680,8 @@
                          #
                          # Example: `admin_listen = 127.0.0.1:8444 http2 ssl`
 
-#status_listen = off     # Comma-separated list of addresses and ports on
+#status_listen = 127.0.0.1:8007 reuseport backlog=16384
+                         # Comma-separated list of addresses and ports on
                          # which the Status API should listen.
                          # The Status API is a read-only endpoint
                          # allowing monitoring tools to retrieve metrics,

kong/clustering/compat/checkers.lua

@@ -23,6 +23,28 @@ end
 
 
 local compatible_checkers = {
+  { 3006000000, --[[ 3.6.0.0 ]]
+    function(config_table, dp_version, log_suffix)
+      local has_update
+      local redis_plugins_update = {
+        acme = require("kong.plugins.acme.clustering.compat.redis_translation").adapter
+      }
+
+      for _, plugin in ipairs(config_table.plugins or {}) do
+        local adapt_fn = redis_plugins_update[plugin.name]
+        if adapt_fn and type(adapt_fn) == "function" then
+          has_update = adapt_fn(plugin.config)
+          if has_update then
+            log_warn_message('adapts ' .. plugin.name .. ' plugin redis configuration to older version',
+            'revert to older schema',
+            dp_version, log_suffix)
+          end
+        end
+      end
+
+      return has_update
+    end,
+  },
   { 3005000000, --[[ 3.5.0.0 ]]
     function(config_table, dp_version, log_suffix)
       local has_update

kong/db/schema/entities/routes.lua

@@ -4,13 +4,14 @@ local deprecation = require("kong.deprecation")
 
 local validate_route
 do
-  local get_schema     = require("kong.router.atc").schema
+  local get_schema = require("kong.router.atc").schema
   local get_expression = require("kong.router.compat").get_expression
+  local transform_expression = require("kong.router.expressions").transform_expression
 
   -- works with both `traditional_compatiable` and `expressions` routes`
   validate_route = function(entity)
     local schema = get_schema(entity.protocols)
-    local exp = entity.expression or get_expression(entity)
+    local exp = transform_expression(entity) or get_expression(entity)
 
     local ok, err = router.validate(schema, exp)
     if not ok then

kong/globalpatches.lua

@@ -1,5 +1,3 @@
-local constants = require "kong.constants"
-
 local ran_before
 
 
@@ -15,6 +13,7 @@ return function(options)
 
   options = options or {}
   local meta = require "kong.meta"
+  local constants = require "kong.constants"
 
 
   local cjson_safe = require("cjson.safe")

kong/plugins/acme/client.lua

@@ -2,6 +2,7 @@ local acme = require "resty.acme.client"
 local util = require "resty.acme.util"
 local x509 = require "resty.openssl.x509"
 local reserved_words = require "kong.plugins.acme.reserved_words"
+local config_adapters = require "kong.plugins.acme.storage.config_adapters"
 
 local cjson = require "cjson"
 local ngx_ssl = require "ngx.ssl"
@@ -82,7 +83,7 @@ local function new_storage_adapter(conf)
   if not storage then
     return nil, nil, "storage is nil"
   end
-  local storage_config = conf.storage_config[storage]
+  local storage_config = config_adapters.adapt_config(conf.storage, conf.storage_config)
   if not storage_config then
     return nil, nil, storage .. " is not defined in plugin storage config"
   end
@@ -101,6 +102,7 @@ local function new(conf)
   if err then
     return nil, err
   end
+  local storage_config = config_adapters.adapt_config(conf.storage, conf.storage_config)
   local account_name = account_name(conf)
   local account, err = cached_get(st, account_name, deserialize_account)
   if err then
@@ -125,7 +127,7 @@ local function new(conf)
     account_key = account.key,
     api_uri = url,
     storage_adapter = storage_full_path,
-    storage_config = conf.storage_config[conf.storage],
+    storage_config = storage_config,
     eab_kid = conf.eab_kid,
     eab_hmac_key = conf.eab_hmac_key,
     challenge_start_callback = hybrid_mode and function()

kong/plugins/acme/clustering/compat/redis_translation.lua

@@ -0,0 +1,23 @@
+local function adapter(config_to_update)
+    if config_to_update.storage == "redis" then
+        config_to_update.storage_config.redis = {
+            host = config_to_update.storage_config.redis.host,
+            port = config_to_update.storage_config.redis.port,
+            auth = config_to_update.storage_config.redis.password,
+            database = config_to_update.storage_config.redis.database,
+            ssl = config_to_update.storage_config.redis.ssl,
+            ssl_verify = config_to_update.storage_config.redis.ssl_verify,
+            ssl_server_name = config_to_update.storage_config.redis.server_name,
+            namespace = config_to_update.storage_config.redis.extra_options.namespace,
+            scan_count = config_to_update.storage_config.redis.extra_options.scan_count
+        }
+
+        return true
+    end
+
+    return false
+end
+
+return {
+    adapter = adapter
+}

kong/plugins/acme/migrations/003_350_to_360.lua

@@ -0,0 +1,41 @@
+return {
+    postgres = {
+      up = [[
+        DO $$
+        BEGIN
+          UPDATE plugins
+          SET config =
+            config
+                #- '{storage_config,redis}'
+
+            || jsonb_build_object(
+                'storage_config',
+                (config -> 'storage_config') - 'redis'
+                || jsonb_build_object(
+                  'redis',
+                  jsonb_build_object(
+                      'host', config #> '{storage_config, redis, host}',
+                      'port', config #> '{storage_config, redis, port}',
+                      'password', config #> '{storage_config, redis, auth}',
+                      'username', config #> '{storage_config, redis, username}',
+                      'ssl', config #> '{storage_config, redis, ssl}',
+                      'ssl_verify', config #> '{storage_config, redis, ssl_verify}',
+                      'server_name', config #> '{storage_config, redis, ssl_server_name}',
+                      'timeout', config #> '{storage_config, redis, timeout}',
+                      'database', config #> '{storage_config, redis, database}'
+                  ) || jsonb_build_object(
+                      'extra_options',
+                      jsonb_build_object(
+                          'scan_count', config #> '{storage_config, redis, scan_count}',
+                          'namespace', config #> '{storage_config, redis, namespace}'
+                      )
+                  )
+                )
+              )
+            WHERE name = 'acme';
+        EXCEPTION WHEN UNDEFINED_COLUMN OR UNDEFINED_TABLE THEN
+          -- Do nothing, accept existing state
+        END$$;
+      ]],
+    },
+}

kong/plugins/acme/migrations/init.lua

@@ -2,4 +2,5 @@ return {
   "000_base_acme",
   "001_280_to_300",
   "002_320_to_330",
+  "003_350_to_360",
 }

kong/plugins/acme/schema.lua

@@ -1,5 +1,9 @@
 local typedefs = require "kong.db.schema.typedefs"
 local reserved_words = require "kong.plugins.acme.reserved_words"
+local redis_schema = require "kong.tools.redis.schema"
+local deprecation = require("kong.deprecation")
+
+local tablex = require "pl.tablex"
 
 local CERT_TYPES = { "rsa", "ecc" }
 
@@ -34,13 +38,9 @@ local SHM_STORAGE_SCHEMA = {
 local KONG_STORAGE_SCHEMA = {
 }
 
-local REDIS_STORAGE_SCHEMA = {
-  { host = typedefs.host, },
-  { port = typedefs.port, },
-  { database = { type = "number", description = "The index of the Redis database to use.", } },
+-- deprecated old schema
+local REDIS_LEGACY_SCHEMA_FIELDS = {
   { auth = { type = "string", referenceable = true, description = "The Redis password to use for authentication. " } },
-  { ssl = { type = "boolean", required = true, default = false, description = "Whether to use SSL/TLS encryption when connecting to the Redis server."} },
-  { ssl_verify = { type = "boolean", required = true, default = false, description = "Whether to verify the SSL/TLS certificate presented by the Redis server. This should be a boolean value." } },
   { ssl_server_name = typedefs.sni { required = false, description = "The expected server name for the SSL/TLS certificate presented by the Redis server." }},
   {
     namespace = {
@@ -55,6 +55,29 @@ local REDIS_STORAGE_SCHEMA = {
   { scan_count = { type = "number", required = false, default = 10, description = "The number of keys to return in Redis SCAN calls." } },
 }
 
+local REDIS_STORAGE_SCHEMA = tablex.copy(redis_schema.config_schema.fields)
+for _,v in ipairs(REDIS_LEGACY_SCHEMA_FIELDS) do
+  table.insert(REDIS_STORAGE_SCHEMA, v)
+end
+
+table.insert(REDIS_STORAGE_SCHEMA, { extra_options = {
+  description = "Custom ACME Redis options",
+  type = "record",
+  fields = {
+    {
+      namespace = {
+        type = "string",
+        description = "A namespace to prepend to all keys stored in Redis.",
+        required = true,
+        default = "",
+        len_min = 0,
+        custom_validator = validate_namespace
+      }
+    },
+    { scan_count = { type = "number", required = false, default = 10, description = "The number of keys to return in Redis SCAN calls." } },
+  }
+} })
+
 local CONSUL_STORAGE_SCHEMA = {
   { https = { type = "boolean", default = false, description = "Boolean representation of https."}, },
   { host = typedefs.host},
@@ -248,6 +271,28 @@ local schema = {
         end
       }
     },
+    { custom_entity_check = {
+      field_sources = { "config.storage_config.redis.namespace", "config.storage_config.redis.scan_count", "config.storage_config.redis.auth", "config.storage_config.redis.ssl_server_name" },
+      fn = function(entity)
+        if (entity.config.storage_config.redis.namespace or ngx.null) ~= ngx.null and entity.config.storage_config.redis.namespace ~= "" then
+          deprecation("acme: config.storage_config.redis.namespace is deprecated, please use config.storage_config.redis.extra_options.namespace instead",
+            { after = "4.0", })
+        end
+        if (entity.config.storage_config.redis.scan_count or ngx.null) ~= ngx.null and entity.config.storage_config.redis.scan_count ~= 10 then
+          deprecation("acme: config.storage_config.redis.scan_count is deprecated, please use config.storage_config.redis.extra_options.scan_count instead",
+            { after = "4.0", })
+        end
+        if (entity.config.storage_config.redis.auth or ngx.null) ~= ngx.null  then
+          deprecation("acme: config.storage_config.redis.auth is deprecated, please use config.storage_config.redis.password instead",
+            { after = "4.0", })
+        end
+        if (entity.config.storage_config.redis.ssl_server_name or ngx.null) ~= ngx.null  then
+          deprecation("acme: config.storage_config.redis.ssl_server_name is deprecated, please use config.storage_config.redis.server_name instead",
+            { after = "4.0", })
+        end
+        return true
+      end
+    } }
   },
 }
 

kong/plugins/acme/storage/config_adapters/init.lua

@@ -0,0 +1,28 @@
+local redis_config_adapter = require "kong.plugins.acme.storage.config_adapters.redis"
+
+local function load_adapters()
+    local adapters_mapping = {
+        redis = redis_config_adapter
+    }
+
+    local function identity(config)
+        return config
+    end
+
+    local default_value_mt = { __index = function() return identity  end }
+
+    setmetatable(adapters_mapping, default_value_mt)
+
+    return adapters_mapping
+end
+
+local adapters = load_adapters()
+
+local function adapt_config(storage_type, storage_config)
+    local adapter_fn = adapters[storage_type]
+    return adapter_fn(storage_config[storage_type])
+end
+
+return {
+    adapt_config = adapt_config
+}

kong/plugins/acme/storage/config_adapters/redis.lua

@@ -0,0 +1,16 @@
+local function redis_config_adapter(conf)
+    return {
+        host = conf.host,
+        port = conf.port,
+        database = conf.database,
+        auth = conf.password or conf.auth, -- allow conf.auth until 4.0 version
+        ssl = conf.ssl,
+        ssl_verify = conf.ssl_verify,
+        ssl_server_name = conf.server_name or conf.ssl_server_name, -- allow conf.ssl_server_name until 4.0 version
+
+        namespace = conf.extra_options.namespace or conf.namespace, -- allow conf.namespace until 4.0 version
+        scan_count = conf.extra_options.scan_count or conf.scan_count, -- allow conf.scan_count until 4.0 version
+    }
+end
+
+return redis_config_adapter

kong/router/atc.lua

@@ -66,7 +66,10 @@ do
                   "http.queries.*",
                  },
 
-    ["Int"]    = {"net.port",
+    ["Int"]    = {"net.src.port", "net.dst.port",
+                 },
+
+    ["IpAddr"] = {"net.src.ip", "net.dst.ip",
                  },
   }
 
@@ -404,8 +407,8 @@ function _M:matching(params)
   local req_host = params.host
 
   check_select_params(params.method, req_uri, req_host, params.scheme,
-                      nil, nil,
-                      nil, nil,
+                      params.src_ip, params.src_port,
+                      params.dst_ip, params.dst_port,
                       params.sni, params.headers, params.queries)
 
   local host, port = split_host_port(req_host)
@@ -464,8 +467,8 @@ end
 
 -- only for unit-testing
 function _M:select(req_method, req_uri, req_host, req_scheme,
-                   _, _,
-                   _, _,
+                   src_ip, src_port,
+                   dst_ip, dst_port,
                    sni, req_headers, req_queries)
 
   local params = {
@@ -476,6 +479,11 @@ function _M:select(req_method, req_uri, req_host, req_scheme,
     sni     = sni,
     headers = req_headers,
     queries = req_queries,
+
+    src_ip   = src_ip,
+    src_port = src_port,
+    dst_ip   = dst_ip,
+    dst_port = dst_port,
   }
 
   return self:matching(params)