New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(clustering) retiring Serf - new cache invalidation mechanism #2561

Merged
merged 12 commits into from Jun 16, 2017
View
@@ -14,7 +14,6 @@ dependencies = {
"luasec == 0.6",
"luasocket == 2.0.2",
"penlight == 1.4.1",
"mediator_lua == 1.1.2",
"lua-resty-http == 0.08",
"lua-resty-jit-uuid == 0.0.5",
"multipart == 0.5",
@@ -37,11 +36,18 @@ build = {
modules = {
["kong"] = "kong/kong.lua",
["kong.meta"] = "kong/meta.lua",
["kong.serf"] = "kong/serf.lua",
["kong.constants"] = "kong/constants.lua",
["kong.singletons"] = "kong/singletons.lua",
["kong.conf_loader"] = "kong/conf_loader.lua",
["kong.cluster_events"] = "kong/cluster_events.lua",
["kong.cluster_events.strategies.cassandra"] = "kong/cluster_events/strategies/cassandra.lua",
["kong.cluster_events.strategies.postgres"] = "kong/cluster_events/strategies/postgres.lua",
["kong.cache"] = "kong/cache.lua",
["kong.mlcache"] = "kong/mlcache.lua",
["kong.templates.nginx"] = "kong/templates/nginx.lua",
["kong.templates.nginx_kong"] = "kong/templates/nginx_kong.lua",
["kong.templates.kong_defaults"] = "kong/templates/kong_defaults.lua",
@@ -56,14 +62,12 @@ build = {
["kong.cmd.check"] = "kong/cmd/check.lua",
["kong.cmd.reload"] = "kong/cmd/reload.lua",
["kong.cmd.restart"] = "kong/cmd/restart.lua",
["kong.cmd.cluster"] = "kong/cmd/cluster.lua",
["kong.cmd.compile"] = "kong/cmd/compile.lua",
["kong.cmd.migrations"] = "kong/cmd/migrations.lua",
["kong.cmd.health"] = "kong/cmd/health.lua",
["kong.cmd.version"] = "kong/cmd/version.lua",
["kong.cmd.utils.log"] = "kong/cmd/utils/log.lua",
["kong.cmd.utils.kill"] = "kong/cmd/utils/kill.lua",
["kong.cmd.utils.serf_signals"] = "kong/cmd/utils/serf_signals.lua",
["kong.cmd.utils.nginx_signals"] = "kong/cmd/utils/nginx_signals.lua",
["kong.cmd.utils.prefix_handler"] = "kong/cmd/utils/prefix_handler.lua",
@@ -75,7 +79,6 @@ build = {
["kong.api.routes.consumers"] = "kong/api/routes/consumers.lua",
["kong.api.routes.plugins"] = "kong/api/routes/plugins.lua",
["kong.api.routes.cache"] = "kong/api/routes/cache.lua",
["kong.api.routes.cluster"] = "kong/api/routes/cluster.lua",
["kong.api.routes.upstreams"] = "kong/api/routes/upstreams.lua",
["kong.api.routes.certificates"] = "kong/api/routes/certificates.lua",
["kong.api.routes.snis"] = "kong/api/routes/snis.lua",
@@ -88,24 +91,19 @@ build = {
["kong.tools.printable"] = "kong/tools/printable.lua",
["kong.tools.responses"] = "kong/tools/responses.lua",
["kong.tools.timestamp"] = "kong/tools/timestamp.lua",
["kong.tools.database_cache"] = "kong/tools/database_cache.lua",
["kong.core.handler"] = "kong/core/handler.lua",
["kong.core.certificate"] = "kong/core/certificate.lua",
["kong.core.router"] = "kong/core/router.lua",
["kong.core.plugins_iterator"] = "kong/core/plugins_iterator.lua",
["kong.core.hooks"] = "kong/core/hooks.lua",
["kong.core.reports"] = "kong/core/reports.lua",
["kong.core.cluster"] = "kong/core/cluster.lua",
["kong.core.events"] = "kong/core/events.lua",
["kong.core.error_handlers"] = "kong/core/error_handlers.lua",
["kong.core.globalpatches"] = "kong/core/globalpatches.lua",
["kong.core.balancer"] = "kong/core/balancer.lua",
["kong.dao.errors"] = "kong/dao/errors.lua",
["kong.dao.schemas_validation"] = "kong/dao/schemas_validation.lua",
["kong.dao.schemas.apis"] = "kong/dao/schemas/apis.lua",
["kong.dao.schemas.nodes"] = "kong/dao/schemas/nodes.lua",
["kong.dao.schemas.consumers"] = "kong/dao/schemas/consumers.lua",
["kong.dao.schemas.plugins"] = "kong/dao/schemas/plugins.lua",
["kong.dao.schemas.upstreams"] = "kong/dao/schemas/upstreams.lua",
@@ -129,14 +127,12 @@ build = {
["kong.plugins.basic-auth.handler"] = "kong/plugins/basic-auth/handler.lua",
["kong.plugins.basic-auth.access"] = "kong/plugins/basic-auth/access.lua",
["kong.plugins.basic-auth.schema"] = "kong/plugins/basic-auth/schema.lua",
["kong.plugins.basic-auth.hooks"] = "kong/plugins/basic-auth/hooks.lua",
["kong.plugins.basic-auth.api"] = "kong/plugins/basic-auth/api.lua",
["kong.plugins.basic-auth.daos"] = "kong/plugins/basic-auth/daos.lua",
["kong.plugins.key-auth.migrations.cassandra"] = "kong/plugins/key-auth/migrations/cassandra.lua",
["kong.plugins.key-auth.migrations.postgres"] = "kong/plugins/key-auth/migrations/postgres.lua",
["kong.plugins.key-auth.handler"] = "kong/plugins/key-auth/handler.lua",
["kong.plugins.key-auth.hooks"] = "kong/plugins/key-auth/hooks.lua",
["kong.plugins.key-auth.schema"] = "kong/plugins/key-auth/schema.lua",
["kong.plugins.key-auth.api"] = "kong/plugins/key-auth/api.lua",
["kong.plugins.key-auth.daos"] = "kong/plugins/key-auth/daos.lua",
@@ -145,7 +141,6 @@ build = {
["kong.plugins.oauth2.migrations.postgres"] = "kong/plugins/oauth2/migrations/postgres.lua",
["kong.plugins.oauth2.handler"] = "kong/plugins/oauth2/handler.lua",
["kong.plugins.oauth2.access"] = "kong/plugins/oauth2/access.lua",
["kong.plugins.oauth2.hooks"] = "kong/plugins/oauth2/hooks.lua",
["kong.plugins.oauth2.schema"] = "kong/plugins/oauth2/schema.lua",
["kong.plugins.oauth2.daos"] = "kong/plugins/oauth2/daos.lua",
["kong.plugins.oauth2.api"] = "kong/plugins/oauth2/api.lua",
@@ -225,7 +220,6 @@ build = {
["kong.plugins.acl.migrations.postgres"] = "kong/plugins/acl/migrations/postgres.lua",
["kong.plugins.acl.handler"] = "kong/plugins/acl/handler.lua",
["kong.plugins.acl.schema"] = "kong/plugins/acl/schema.lua",
["kong.plugins.acl.hooks"] = "kong/plugins/acl/hooks.lua",
["kong.plugins.acl.api"] = "kong/plugins/acl/api.lua",
["kong.plugins.acl.daos"] = "kong/plugins/acl/daos.lua",
@@ -236,7 +230,6 @@ build = {
["kong.plugins.jwt.migrations.postgres"] = "kong/plugins/jwt/migrations/postgres.lua",
["kong.plugins.jwt.handler"] = "kong/plugins/jwt/handler.lua",
["kong.plugins.jwt.schema"] = "kong/plugins/jwt/schema.lua",
["kong.plugins.jwt.hooks"] = "kong/plugins/jwt/hooks.lua",
["kong.plugins.jwt.api"] = "kong/plugins/jwt/api.lua",
["kong.plugins.jwt.daos"] = "kong/plugins/jwt/daos.lua",
["kong.plugins.jwt.jwt_parser"] = "kong/plugins/jwt/jwt_parser.lua",
@@ -247,7 +240,6 @@ build = {
["kong.plugins.hmac-auth.handler"] = "kong/plugins/hmac-auth/handler.lua",
["kong.plugins.hmac-auth.access"] = "kong/plugins/hmac-auth/access.lua",
["kong.plugins.hmac-auth.schema"] = "kong/plugins/hmac-auth/schema.lua",
["kong.plugins.hmac-auth.hooks"] = "kong/plugins/hmac-auth/hooks.lua",
["kong.plugins.hmac-auth.api"] = "kong/plugins/hmac-auth/api.lua",
["kong.plugins.hmac-auth.daos"] = "kong/plugins/hmac-auth/daos.lua",
@@ -274,8 +266,6 @@ build = {
["kong.plugins.bot-detection.handler"] = "kong/plugins/bot-detection/handler.lua",
["kong.plugins.bot-detection.schema"] = "kong/plugins/bot-detection/schema.lua",
["kong.plugins.bot-detection.rules"] = "kong/plugins/bot-detection/rules.lua",
["kong.plugins.bot-detection.cache"] = "kong/plugins/bot-detection/cache.lua",
["kong.plugins.bot-detection.hooks"] = "kong/plugins/bot-detection/hooks.lua",
["kong.plugins.aws-lambda.handler"] = "kong/plugins/aws-lambda/handler.lua",
["kong.plugins.aws-lambda.schema"] = "kong/plugins/aws-lambda/schema.lua",
View
@@ -365,72 +365,45 @@
# migrations.
#------------------------------------------------------------------------------
# CLUSTERING
# DATASTORE CACHE
#------------------------------------------------------------------------------
# In addition to pointing to the same database, each Kong node must be
# reachable by the other nodes in the cluster.
# In order to avoid unecessary communication with the datastore, Kong caches
# entities (such as APIs, Consumers, Credentials...) for a configurable period
# of time. It also handles invalidations if such an entity is updated.
#
# Kong's clustering works on the IP layer (hostnames are not supported, only
# IPs) and expects a flat network topology without any NAT between the
# datacenters.
#
# A common pattern is to create a VPN between the two datacenters such that
# the flat network assumption is not violated.
#
# See the clustering reference for more informations:
# https://getkong.org/docs/latest/clustering/
#cluster_listen = 0.0.0.0:7946 # Address and port used to communicate with
# other nodes in the cluster.
# All other Kong nodes in the same cluster
# must be able to communicate over both
# TCP and UDP on this address.
# Only IPv4 addresses are supported.
#cluster_listen_rpc = 127.0.0.1:7373 # Address and port used to communicate
# with the cluster through the agent
# running on this node. Only contains
# TCP traffic local to this node.
#cluster_advertise = # By default, the `cluster_listen` address
# is advertised over the cluster.
# If the `cluster_listen` host is '0.0.0.0',
# then the first local, non-loopback IPv4
# address will be advertised to other nodes.
# However, in some cases (specifically NAT
# traversal), there may be a routable address
# that cannot be bound to. This flag enables
# advertising a different address to support
# this.
#cluster_encrypt_key = # base64-encoded 16-bytes key to encrypt
# cluster traffic with.
#cluster_keyring_file = # Specifies a file to load keyring data from.
# Kong is able to keep encryption keys in sync
# and perform key rotations. During a key
# rotation, there may be some period of time
# in which Kong is required to maintain more
# than one encryption key until all members
# have received the new key.
#cluster_ttl_on_failure = 3600 # Time to live (in seconds) of a node in the
# cluster when it stops sending healthcheck
# pings, possibly caused by a node or network
# failure.
# If a node is not able to send a new
# healthcheck ping before the expiration,
# other nodes in the cluster will stop
# attempting to connect to it.
# Recommended to be at least `60`.
#cluster_profile = wan # The timing profile for inter-cluster pings
# and timeouts. If a `lan` or `local` profile
# is used over the Internet, a high rate of
# failures is risked as the timing contraints
# would be too tight.
# Accepted values are `local`, `lan`, `wan`.
# This section allows for configuring the behavior of Kong regarding the
# caching of such configuration entities.
#db_update_frequency = 5 # Frequency (in seconds) at which to check for
# updated entities with the datastore.
# When a node creates, updates, or deletes an
# entity via the Admin API, other nodes need
# to wait for the next poll (configured by
# this value) to eventually purge the old
# cached entity and start using the new one.
#db_update_propagation = 0 # Time (in seconds) taken for an entity in the
# datastore to be propagated to replica nodes
# of another datacenter.
# When in a distributed environment such as
# a multi-datacenter Cassandra cluster, this
# value should be the maximum number of
# seconds taken by Cassandra to propagate a
# row to other datacenters.
# When set, this property will increase the
# time taken by Kong to propagate the change
# of an entity.
# Single-datacenter setups or PostgreSQL
# servers should suffer no such delays, and
# this value can be safely set to 0.

This comment has been minimized.

@Tieske

Tieske Jun 2, 2017

Member

I do not understand what this setting does?

@Tieske

Tieske Jun 2, 2017

Member

I do not understand what this setting does?

This comment has been minimized.

@thibaultcha

thibaultcha Jun 15, 2017

Member

Updated the description to clarify the meaning

@thibaultcha

thibaultcha Jun 15, 2017

Member

Updated the description to clarify the meaning

#db_cache_ttl = 3600 # Time-to-live (in seconds) of an entity from
# the datastore when cached by this node.
# Database misses (no entity) are also cached
# according to this setting.
# If set to 0, such cached entities/misses
# never expire.
#------------------------------------------------------------------------------
# DNS RESOLVER
View
@@ -123,7 +123,7 @@ ngx.log(ngx.DEBUG, "Loading Admin API endpoints")
-- Load core routes
for _, v in ipairs({"kong", "apis", "consumers", "plugins", "cache", "cluster",
for _, v in ipairs({"kong", "apis", "consumers", "plugins", "cache",
"certificates", "snis", "upstreams"}) do
local routes = require("kong.api.routes." .. v)
attach_routes(routes)
View
@@ -1,77 +1,34 @@
local responses = require "kong.tools.responses"
local cache = require "kong.tools.database_cache"
local singletons = require "kong.singletons"
-- Create a table of functions per cache type
local caches
do
local cache_names = {}
caches = setmetatable({
lua = function(action, ...)
if action == "get" then
return cache.get(...)
elseif action == "delete" then
return cache.delete(...)
elseif action == "delete_all" then
return cache.delete_all(...)
end
end,
shm = function(action, ...)
if action == "get" then
return cache.sh_get(...)
elseif action == "delete" then
return cache.sh_delete(...)
elseif action == "delete_all" then
return cache.sh_delete_all(...)
return {
["/cache/:key"] = {
GET = function(self, _, helpers)
-- probe the cache to see if a key has been requested before
local ttl, err, value = singletons.cache:probe(self.params.key)
if err then
return helpers.responses.send_HTTP_INTERNAL_SERVER_ERROR(err)
end
end,
}, {
__index = function(self, key)
return responses.send_HTTP_BAD_REQUEST("invalid cache type; '" ..
tostring(key) .. "', valid caches are: " .. cache_names)
end,
})
-- build string with valid cache names (for error mesage above)
for name in pairs(caches) do cache_names[#cache_names+1] = "'" .. name .. "'" end
table.sort(cache_names) -- make order deterministic, for test purposes
cache_names = table.concat(cache_names, ", ")
end
if ttl then
return helpers.responses.send_HTTP_OK(value)
end
return {
["/cache/"] = {
before = function(self)
self.params.cache = self.params.cache or "lua"
return helpers.responses.send_HTTP_NOT_FOUND()
end,
DELETE = function(self, dao_factory)
caches[self.params.cache]("delete_all")
return responses.send_HTTP_NO_CONTENT()
end
},
DELETE = function(self, _, helpers)
singletons.cache:invalidate_local(self.params.key)
["/cache/:key"] = {
before = function(self)
self.params.cache = self.params.cache or "lua"
return helpers.responses.send_HTTP_NO_CONTENT()
end,
},
GET = function(self, dao_factory)
if self.params.key then
local cached_item = caches[self.params.cache]("get", self.params.key)
if cached_item ~= nil then
return responses.send_HTTP_OK(cached_item)
end
end
["/cache"] = {
DELETE = function(self, _, helpers)
singletons.cache:purge()
return responses.send_HTTP_NOT_FOUND()
return helpers.responses.send_HTTP_NO_CONTENT()
end,
DELETE = function(self, dao_factory)
if self.params.key then
caches[self.params.cache]("delete", self.params.key)
return responses.send_HTTP_NO_CONTENT()
end
return responses.send_HTTP_NOT_FOUND()
end
}
},
}
Oops, something went wrong.
ProTip! Use n and p to navigate between commits in a pull request.