diff --git a/.gitignore b/.gitignore index 21cef7e3f5d..0bcf9c3dc87 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ .project .idea +servroot* + # kong nginx_tmp/ kong*.yml diff --git a/bin/kong b/bin/kong index 55e12b175a5..03312e22f40 100755 --- a/bin/kong +++ b/bin/kong @@ -1,54 +1,3 @@ -#!/usr/bin/env luajit +#!/usr/bin/env resty --- Kong CLI entry-point (bin/kong). --- --- Kong's CLI is a set of small commands invoked by a global executable, this file. --- --- All commands are invoked by this script, then parsed (arguments and options) --- by lapp (see http://lua-users.org/wiki/LappFramework). --- --- This script is not parsed by lapp due to limitations of the said framework as it --- is currently implemented. - -local meta = require "kong.meta" -local commands = { - db = "kong.cli.cmds.db", - stop = "kong.cli.cmds.stop", - quit = "kong.cli.cmds.quit", - start = "kong.cli.cmds.start", - reload = "kong.cli.cmds.reload", - config = "kong.cli.cmds.config", - restart = "kong.cli.cmds.restart", - version = "kong.cli.cmds.version", - status = "kong.cli.cmds.status", - migrations = "kong.cli.cmds.migrations", - cluster = "kong.cli.cmds.cluster", - ["--version"] = "kong.cli.cmds.version" -} - -local help_message = string.format([[ -Usage: kong - - where is one of: - start, restart, reload, stop, quit, cluster, status, migrations, version - - kong --help print this message - kong --help print the help message of a command - -%s@%s]], meta._NAME, meta._VERSION) - --- Determine validity of the given command -local cmd = arg[1] -if not cmd then - print("Missing \n\n"..help_message) - os.exit(1) -elseif cmd == "-h" or cmd == "--help" then - print(help_message) - os.exit(0) -elseif not commands[cmd] then - print("Invalid : "..cmd.."\n\n"..help_message) - os.exit(1) -end - --- Load and execute desired command -require(commands[cmd]) +require("kong.cmd.init")(arg) diff --git a/kong/cli/services/serf.lua b/kong/cli/services/serf.lua index 5db39e99562..f912eee55e0 100644 --- a/kong/cli/services/serf.lua +++ b/kong/cli/services/serf.lua @@ -12,14 +12,8 @@ local SERVICE_NAME = "serf" local START_TIMEOUT = 10 local EVENT_NAME = "kong" -function Serf:new(configuration) - local nginx_working_dir = configuration.nginx_working_dir - - self._configuration = configuration - local path_prefix = nginx_working_dir - ..(stringy.endswith(nginx_working_dir, "/") and "" or "/") - self._script_path = path_prefix.."serf_event.sh" - self._log_path = path_prefix.."serf.log" +function Serf:new(kong_config) + self._configuration = kong_config self._dao_factory = dao_loader.load(self._configuration) Serf.super.new(self, SERVICE_NAME, nginx_working_dir) end @@ -37,47 +31,6 @@ function Serf:_get_cmd() return cmd, err end -function Serf:prepare() - -- Create working directory if missing - local ok, err = Serf.super.prepare(self, self._configuration.nginx_working_dir) - if not ok then - return nil, err - end - - -- Create serf event handler - local luajit_path = BaseService.find_cmd("luajit") - if not luajit_path then - return nil, "Can't find luajit" - end - - local script = [[ -#!/bin/sh -PAYLOAD=`cat` # Read from stdin - -if [ "$SERF_EVENT" != "user" ]; then - PAYLOAD="{\"type\":\"${SERF_EVENT}\",\"entity\": \"${PAYLOAD}\"}" -fi - -echo $PAYLOAD > /tmp/payload - -COMMAND='require("kong.tools.http_client").post("http://]]..self._configuration.admin_api_listen..[[/cluster/events/", ]].."[=['${PAYLOAD}']=]"..[[, {["content-type"] = "application/json"})' - -echo $COMMAND | ]]..luajit_path..[[ -]] - local _, err = IO.write_to_file(self._script_path, script) - if err then - return false, err - end - - -- Adding executable permissions - local res, code = IO.os_execute("chmod +x "..self._script_path) - if code ~= 0 then - return false, res - end - - return true -end - function Serf:_join_node(address) local _, err = self:invoke_signal("join", {address}) if err then @@ -260,21 +213,4 @@ function Serf:event(t_payload) return self:invoke_signal("event "..tostring(args).." kong", {"'"..encoded_payload.."'", "&"}, true) end -function Serf:stop() - logger:info("Leaving cluster..") - local _, err = self:invoke_signal("leave") - if err then - return false, err - else - -- Remove the node from the datastore. - -- This is useful when this is the only node running in the cluster. - self._dao_factory.nodes:delete({ - name = cluster_utils.get_node_name(self._configuration) - }) - - -- Finally stop Serf - Serf.super.stop(self, true) - end -end - return Serf diff --git a/kong/cmd/init.lua b/kong/cmd/init.lua new file mode 100644 index 00000000000..842f84c870e --- /dev/null +++ b/kong/cmd/init.lua @@ -0,0 +1,56 @@ +local pl_app = require "pl.lapp" +local help = [[ +Kong, open-source API gateway. + +Usage: kong COMMAND [OPTIONS] + +The available commands are: + start + stop + +Options: + --trace (optional boolean) with traceback +]] + +local DEFAULT_NGINX_PREFIX = "servroot" + +local cmds = { + start = "start", + stop = "stop", + --reload = "reload", + --migrate = "migrate", + --reset = "reset" +} + +return function(args) + local cmd_name = args[1] + if cmd_name == nil then + pl_app(help) + pl_app.quit() + elseif not cmds[cmd_name] then + pl_app(help) + pl_app.quit("No such command: "..cmd_name) + end + + local cmd = require("kong.cmd."..cmd_name) + local cmd_lapp = cmd.lapp + local cmd_exec = cmd.execute + + cmd_lapp = cmd_lapp.."\n --trace (optional boolean) with traceback\n" + args = pl_app(cmd_lapp) + args.prefix = args.prefix or DEFAULT_NGINX_PREFIX + + xpcall(function() cmd_exec(args) end, function(err) + if not args.trace then + err = err:match "^.-:.-:.(.*)$" + io.stderr:write("Error: "..err.."\n") + io.stderr:write("\n Run with --trace to see traceback\n") + else + local trace = debug.traceback(err, 2) + io.stderr:write("Error: \n") + io.stderr:write(trace.."\n") + end + + pl_app.quit(nil, true) + end) +end diff --git a/kong/cmd/start.lua b/kong/cmd/start.lua new file mode 100644 index 00000000000..b0ff8894eb9 --- /dev/null +++ b/kong/cmd/start.lua @@ -0,0 +1,26 @@ +local nginx_conf_compiler = require "kong.cmd.utils.nginx_conf_compiler" +local nginx_signals = require "kong.cmd.utils.nginx_signals" +local serf_signals = require "kong.cmd.utils.serf_signals" +local conf_loader = require "kong.conf_loader" +local DAOFactory = require "kong.dao.factory" + +local function execute(args) + local conf = assert(conf_loader(args.conf)) + assert(nginx_conf_compiler.prepare_prefix(conf, args.prefix)) + assert(serf_signals.start(conf, args.prefix, DAOFactory(conf))) + assert(nginx_signals.start(args.prefix)) + print("Started") +end + +local lapp = [[ +Usage: kong start [OPTIONS] + +Options: + -c,--conf (optional string) configuration file + --prefix (optional string) Nginx prefix path +]] + +return { + lapp = lapp, + execute = execute +} diff --git a/kong/cmd/stop.lua b/kong/cmd/stop.lua new file mode 100644 index 00000000000..87266433d98 --- /dev/null +++ b/kong/cmd/stop.lua @@ -0,0 +1,20 @@ +local nginx_signals = require "kong.cmd.utils.nginx_signals" +local serf_signals = require "kong.cmd.utils.serf_signals" + +local function execute(args) + assert(nginx_signals.stop(args.prefix)) + assert(serf_signals.stop(args.prefix)) + print("Stopped") +end + +local lapp = [[ +Usage: kong stop [OPTIONS] + +Options: + --prefix (optional string) Nginx prefix path +]] + +return { + lapp = lapp, + execute = execute +} diff --git a/kong/cmd/utils/nginx_conf_compiler.lua b/kong/cmd/utils/nginx_conf_compiler.lua index 3e5382a8d4d..32cd2605d7d 100644 --- a/kong/cmd/utils/nginx_conf_compiler.lua +++ b/kong/cmd/utils/nginx_conf_compiler.lua @@ -24,6 +24,8 @@ local function compile_conf(kong_config, conf_template) if kong_config.cassandra_ssl and kong_config.cassandra_ssl_trusted_cert then compile_env["lua_ssl_trusted_certificate"] = kong_config.cassandra_ssl_trusted_cert + --compile_env["ssl_certificate"] = + --compile_env["ssl_certificate_key"] = end if kong_config.nginx_optimizations then diff --git a/kong/cmd/utils/nginx_signals.lua b/kong/cmd/utils/nginx_signals.lua new file mode 100644 index 00000000000..45cc223fbea --- /dev/null +++ b/kong/cmd/utils/nginx_signals.lua @@ -0,0 +1,80 @@ +local pl_utils = require "pl.utils" +local pl_path = require "pl.path" +local pl_file = require "pl.file" +local fmt = string.format + +local nginx_bin_name = "nginx" +local nginx_search_paths = { + "/usr/local/openresty/nginx/sbin", + "" +} + +local function is_openresty(bin_path) + local cmd = fmt("%s -v", bin_path) + local ok, _, _, v_str = pl_utils.executeex(cmd) + if ok and v_str then + return v_str:match "^nginx version: ngx_openresty/" or + v_str:match "^nginx version: openresty/" + end +end + +local function get_pid(nginx_prefix) + local pid_path = pl_path.join(nginx_prefix, "logs", "nginx.pid") + if pl_path.exists(pid_path) then + return pl_file.read(pid_path) + end +end + +local function send_signal(nginx_prefix, signal) + local pid = get_pid(nginx_prefix) + if not pid then return nil, "could not get Nginx pid (is Kong running?)" end + + local cmd = fmt("kill -s %s %s", signal, pid) + + local ok = pl_utils.execute(cmd) + if not ok then return nil, "could not send signal" end + + return true +end + +local _M = {} + +function _M.find_bin() + local found + for _, path in ipairs(nginx_search_paths) do + local path_to_check = pl_path.join(path, nginx_bin_name) + local ok = is_openresty(path_to_check) + if ok then + found = path_to_check + break + end + end + + if not found then + return nil, "could not find OpenResty 'nginx' executable" + end + + return found +end + +function _M.start(nginx_prefix) + local nginx_bin, err = _M.find_bin() + if not nginx_bin then return nil, err end + + local cmd = fmt("%s -p %s -c %s", nginx_bin, nginx_prefix, "nginx.conf") + + local ok, _, _, stderr = pl_utils.executeex(cmd) + if not ok then return nil, stderr end + + return true +end + +function _M.stop(nginx_prefix) + return send_signal(nginx_prefix, "QUIT") +end + +function _M.reload(nginx_prefix) + return send_signal(nginx_prefix, "HUP") +end + +return _M diff --git a/kong/cmd/utils/serf_signals.lua b/kong/cmd/utils/serf_signals.lua new file mode 100644 index 00000000000..54a2b5094b8 --- /dev/null +++ b/kong/cmd/utils/serf_signals.lua @@ -0,0 +1,147 @@ +-- Enhanced implementation of previous "services.serf.lua" module, +-- no change in acutal logic, only decoupled from the events features +-- which now live in kong.serf + +local Serf = require "kong.serf" + +local pl_stringx = require "pl.stringx" +local pl_utils = require "pl.utils" +local pl_path = require "pl.path" +local pl_file = require "pl.file" +local fmt = string.format + +local serf_bin_name = "serf" +local serf_pid_name = "serf.pid" +local serf_event_name = "kong" +local start_timeout = 2 + +local function check_serf_bin() + local cmd = fmt("%s -v", serf_bin_name) + local ok, _, stdout = pl_utils.executeex(cmd) + if ok and stdout then + if not stdout:match "^Serf v0%.7%.0" then + return nil, "wrong Serf version (need 0.7.0)" + end + return true + end + + return nil, "could not find Serf executable (is it in your $PATH?)" +end + +-- script from old services.serf module +local script_template = [[ +#!/bin/sh + +PAYLOAD=`cat` # Read from stdin +if [ "$SERF_EVENT" != "user" ]; then + PAYLOAD="{\"type\":\"${SERF_EVENT}\",\"entity\": \"${PAYLOAD}\"}" +fi +echo $PAYLOAD > /tmp/payload + +resty -e "require('kong.tools.http_client').post('http://%s/cluster/events/', ${PAYLOAD}, {['content-type'] = 'application/json'})" + +exit 0 +]] + +local function prepare_prefix(kong_config, nginx_prefix, script_path) + local script = fmt(script_template, kong_config.admin_listen) + + pl_file.write(script_path, script) + local ok, _, _, stderr = pl_utils.executeex("chmod +x "..script_path) + if not ok then return nil, stderr end + + return true +end + +local function is_running(pid_path) + if not pl_path.exists(pid_path) then return nil end + local pid = pl_file.read(pid_path) + if pid == "" then return nil end + + local ok, code = pl_utils.execute(fmt("kill -0 %s", pid)) + return ok and code == 0, pid +end + +local _M = {} + +function _M.start(kong_config, nginx_prefix, dao) + -- is Serf already running in this prefix? + local pid_path = pl_path.join(nginx_prefix, serf_pid_name) + if is_running(pid_path) then + return true + else + pl_file.delete(pid_path) + end + + -- make sure Serf is in PATH + local ok, err = check_serf_bin() + if not ok then return nil, err end + + local serf = Serf.new(kong_config, dao) + + local node_name = serf.node_name + local script_path = pl_path.join(nginx_prefix, "serf_event.sh") + local log_path = pl_path.join(nginx_prefix, "serf.log") + + -- prepare shell script + local ok, err = prepare_prefix(kong_config, nginx_prefix, script_path) + if not ok then return nil, err end + + local args = setmetatable({ + ["-bind"] = kong_config.cluster_listen, + ["-rpc-addr"] = kong_config.cluster_listen_rpc, + ["-advertise"] = kong_config.cluster_advertise, + ["-encrypt"] = kong_config.cluster_encrypt, + ["-log-level"] = "err", + ["-profile"] = "wan", + ["-node"] = node_name, + ["-event-handler"] = "member-join,member-leave,member-failed," + .."member-update,member-reap,user:" + ..serf_event_name.."="..script_path + }, Serf.args_mt) + + local cmd = fmt("nohup %s agent %s > %s 2>&1 & echo $! > %s", + serf_bin_name, tostring(args), + log_path, pid_path) + + -- start Serf agent + local ok = pl_utils.execute(cmd) + if not ok then return nil end + + -- ensure started (just an improved version of previous Serf service) + local tstart = ngx.time() + local texp, started = tstart + start_timeout + + repeat + ngx.sleep "0.2" + started = is_running(pid_path) + until started or ngx.time() >= texp + + if not started then + -- time to get latest error log from serf.log + local logs = pl_file.read(log_path) + local tlogs = pl_stringx.split(logs, "\n") + local err = string.gsub(tlogs[#tlogs-1], "==> ", "") + err = pl_stringx.strip(err) + return nil, "could not start Serf:\n "..err + end + + local ok, err = serf:autojoin() + if not ok then return nil, err end + + local ok, err = serf:add_node() + if not ok then return nil, err end + + return true +end + +function _M.stop(nginx_prefix) + local pid_path = pl_path.join(nginx_prefix, serf_pid_name) + local running, pid = is_running(pid_path) + if not running then return true end + + local ok, code = pl_utils.execute(fmt("kill %s", pid)) + return ok and code == 0 +end + +return _M diff --git a/kong/conf_loader.lua b/kong/conf_loader.lua index 2d464037931..a17a1648887 100644 --- a/kong/conf_loader.lua +++ b/kong/conf_loader.lua @@ -6,12 +6,13 @@ local DEFAULT_PATHS = { local CONF_SCHEMA = { -- kong ssl = {typ = "boolean"}, - database = {enum = {"postgres", "cassandra"}}, - dnsmasq = {typ = "boolean"}, - pg_port = {typ = "number"}, + custom_plugins = {typ = "array"}, + database = {enum = {"postgres", "cassandra"}}, + pg_port = {typ = "number"}, cassandra_contact_points = {typ = "array"}, + cassandra_port = {typ = "number"}, cassandra_repl_strategy = {enum = {"SimpleStrategy", "NetworkTopologyStrategy"}}, cassandra_repl_factor = {typ = "number"}, cassandra_data_centers = {typ = "array"}, @@ -21,6 +22,8 @@ local CONF_SCHEMA = { cassandra_ssl = {typ = "boolean"}, cassandra_ssl_verify = {typ = "boolean"}, + dnsmasq = {typ = "boolean"}, + anonymous_reports = {typ = "boolean"}, -- ngx_lua @@ -29,7 +32,7 @@ local CONF_SCHEMA = { -- nginx nginx_daemon = {typ = "ngx_boolean"}, nginx_optimizations = {typ = "boolean"}, - nginx_worker_processes = {typ = "string"} + nginx_worker_processes = {typ = "string"} -- force string inference } local kong_default_conf = require "kong.templates.kong_defaults" @@ -64,6 +67,7 @@ local function overrides(k, default_v, file_conf, arg_conf, conf_schema) end -- transform {boolean} values ("on"/"off" aliasing to true/false) + -- transform {ngx_boolean} values ("on"/"off" aliasing to on/off) -- transform {explicit string} values (number values converted to strings) -- transform {array} values (comma-separated strings) if conf_schema[k] ~= nil then @@ -73,7 +77,7 @@ local function overrides(k, default_v, file_conf, arg_conf, conf_schema) elseif typ == "ngx_boolean" then value = (value == "on" or value == true) and "on" or "off" elseif typ == "string" then - value = tostring(value) + value = tostring(value) -- forced string inference elseif typ == "number" then value = tonumber(value) -- catch ENV variables (strings) that should be numbers elseif typ == "array" and type(value) == "string" then @@ -82,6 +86,9 @@ local function overrides(k, default_v, file_conf, arg_conf, conf_schema) -- only one element) value = setmetatable(pl_stringx.split(value, ","), nil) -- remove List mt end + elseif type(value) == "string" then + -- default type is string, and an empty if unset + value = value ~= "" and tostring(value) or nil end return value, k @@ -111,13 +118,21 @@ local function validate(conf, conf_schema) end end + -- custom validation + if conf.ssl then + if not conf.ssl_cert then + return nil, "ssl_cert required if SSL enabled" + elseif not conf.ssl_cert_key then + return nil, "ssl_cert_key required if SSL enabled" + end + end + return true end -- @param[type=string] path A path to a conf file -- @param[type=table] custom_conf A table taking precedence over all other sources. local function load(path, custom_conf) - ------------------------ -- Default configuration ------------------------ diff --git a/kong/constants.lua b/kong/constants.lua index dc1983b30a4..c8427c7f83a 100644 --- a/kong/constants.lua +++ b/kong/constants.lua @@ -1,3 +1,16 @@ +local plugins = { + "ssl", "jwt", "acl", "correlation-id", "cors", "oauth2", "tcp-log", "udp-log", + "file-log", "http-log", "key-auth", "hmac-auth", "basic-auth", "ip-restriction", + "galileo", "request-transformer", "response-transformer", + "request-size-limiting", "rate-limiting", "response-ratelimiting", "syslog", + "loggly", "datadog", "runscope", "ldap-auth", "statsd" +} + +local plugin_map = {} +for i = 1, #plugins do + plugin_map[plugins[i]] = true +end + return { SYSLOG = { ADDRESS = "kong-hf.mashape.com", @@ -8,14 +21,8 @@ return { GLOBAL_KONG_CONF = "/etc/kong/kong.yml", NGINX_CONFIG = "nginx.conf" }, - PLUGINS_AVAILABLE = { - "ssl", "jwt", "acl", "correlation-id", "cors", "oauth2", "tcp-log", "udp-log", "file-log", - "http-log", "key-auth", "hmac-auth", "basic-auth", "ip-restriction", - "galileo", "request-transformer", "response-transformer", - "request-size-limiting", "rate-limiting", "response-ratelimiting", "syslog", - "loggly", "datadog", "runscope", "ldap-auth", "statsd" - }, - -- Non standard headers, specific to Kong + PLUGINS_AVAILABLE = plugin_map, + -- non-standard headers, specific to Kong HEADERS = { HOST_OVERRIDE = "X-Host-Override", PROXY_LATENCY = "X-Kong-Proxy-Latency", diff --git a/kong/core/cluster.lua b/kong/core/cluster.lua index a87918af983..60faeb42bfd 100644 --- a/kong/core/cluster.lua +++ b/kong/core/cluster.lua @@ -25,8 +25,6 @@ local function async_autojoin(premature) -- If this node is the only node in the cluster, but other nodes are present, then try to join them -- This usually happens when two nodes are started very fast, and the first node didn't write his -- information into the datastore yet. When the second node starts up, there is nothing to join yet. - if not singletons.configuration.cluster["auto-join"] then return end - local lock = resty_lock:new("cluster_autojoin_locks", { exptime = ASYNC_AUTOJOIN_INTERVAL - 0.001 }) diff --git a/kong/dao/cassandra_db.lua b/kong/dao/cassandra_db.lua index 1d8889a609f..6921d36aef4 100644 --- a/kong/dao/cassandra_db.lua +++ b/kong/dao/cassandra_db.lua @@ -20,27 +20,28 @@ CassandraDB.dao_insert_values = { end } -function CassandraDB:new(options) +function CassandraDB:new(kong_config) local conn_opts = { shm = "cassandra", prepared_shm = "cassandra_prepared", - contact_points = options.contact_points, - keyspace = options.keyspace, + contact_points = kong_config.cassandra_contact_points, + keyspace = kong_config.cassandra_keyspace, protocol_options = { - default_port = options.port + default_port = kong_config.cassandra_port }, query_options = { prepare = true }, ssl_options = { - enabled = options.ssl.enabled, - verify = options.ssl.verify, - ca = options.ssl.certificate_authority + enabled = kong_config.cassandra_ssl, + verify = kong_config.cassandra_ssl_verify, + ca = kong_config.cassandra_ssl_trusted_cert } } - if options.username and options.password then - conn_opts.auth = cassandra.auth.PlainTextProvider(options.username, options.password) + if kong_config.cassandra_username and kong_config.cassandra_password then + conn_opts.auth = cassandra.auth.PlainTextProvider(kong_config.cassandra_username, + kong_config.cassandra_password) end CassandraDB.super.new(self, "cassandra", conn_opts) @@ -165,6 +166,7 @@ function CassandraDB:query(query, args, opts, schema, no_keyspace) if no_keyspace then conn_opts.keyspace = nil end + local session, err = cassandra.spawn_session(conn_opts) if err then return nil, Errors.db(tostring(err)) diff --git a/kong/dao/factory.lua b/kong/dao/factory.lua index 9bc2b4a7b7f..3d1078b9989 100644 --- a/kong/dao/factory.lua +++ b/kong/dao/factory.lua @@ -73,25 +73,25 @@ local function load_daos(self, schemas, constraints, events_handler) end end -function Factory:new(db_type, options, plugins, events_handler) - self.db_type = db_type +function Factory:new(kong_config, plugins, events_handler) + self.db_type = kong_config.database self.daos = {} - self.properties = options - self.plugins_names = plugins or {} + self.kong_config = kong_config + self.plugin_names = plugins or {} local schemas = {} - local DB = require("kong.dao."..db_type.."_db") - _db = DB(options) + local DB = require("kong.dao."..self.db_type.."_db") + _db = DB(kong_config) for _, m_name in ipairs(CORE_MODELS) do schemas[m_name] = require("kong.dao.schemas."..m_name) end - for _, plugin_name in ipairs(self.plugins_names) do + for plugin_name in pairs(self.plugin_names) do local has_dao, plugin_daos = utils.load_module_if_exists("kong.plugins."..plugin_name..".dao."..self.db_type) if has_dao then for k, v in pairs(plugin_daos) do - self.daos[k] = v(options) + self.daos[k] = v(kong_config) end end @@ -149,7 +149,7 @@ function Factory:migrations_modules() core = require("kong.dao.migrations."..self.db_type) } - for _, plugin_name in ipairs(self.plugins_names) do + for plugin_name in pairs(self.plugin_names) do local ok, plugin_mig = utils.load_module_if_exists("kong.plugins."..plugin_name..".migrations."..self.db_type) if ok then migrations[plugin_name] = plugin_mig @@ -193,7 +193,7 @@ local function migrate(self, identifier, migrations_modules, cur_migrations, on_ if mig_type == "string" then err = _db:queries(migration.up) elseif mig_type == "function" then - err = migration.up(_db, self.properties, self) + err = migration.up(_db, self.kong_config, self) end if err then diff --git a/kong/dao/postgres_db.lua b/kong/dao/postgres_db.lua index 6fb50933782..3a962614be6 100644 --- a/kong/dao/postgres_db.lua +++ b/kong/dao/postgres_db.lua @@ -20,8 +20,16 @@ PostgresDB.dao_insert_values = { PostgresDB.additional_tables = {"ttls"} -function PostgresDB:new(...) - PostgresDB.super.new(self, "postgres", ...) +function PostgresDB:new(kong_config) + local conn_opts = { + host = kong_config.pg_host, + port = kong_config.pg_port, + user = kong_config.pg_user, + password = kong_config.pg_password, + database = kong_config.pg_database + } + + PostgresDB.super.new(self, "postgres", conn_opts) end -- TTL clean up timer functions @@ -241,7 +249,7 @@ function PostgresDB:ttl(tbl, table_name, schema, ttl) local expire_at = res[1].timestamp + (ttl * 1000) local query = string.format("SELECT upsert_ttl('%s', %s, '%s', '%s', to_timestamp(%d/1000) at time zone 'UTC')", - tbl[schema.primary_key[1]], primary_key_type == "uuid" and "'"..tbl[schema.primary_key[1]].."'" or "NULL", + tbl[schema.primary_key[1]], primary_key_type == "uuid" and "'"..tbl[schema.primary_key[1]].."'" or "NULL", schema.primary_key[1], table_name, expire_at) local _, err = self:query(query) if err then @@ -270,7 +278,7 @@ function PostgresDB:clear_expired_ttl() return false, err end end - + return true end diff --git a/kong/kong.lua b/kong/kong.lua index ef612fb701d..36fa8593d82 100644 --- a/kong/kong.lua +++ b/kong/kong.lua @@ -32,94 +32,93 @@ _G._KONG = { } local core = require "kong.core.handler" -local Serf = require "kong.cli.services.serf" +local Serf = require "kong.serf" local utils = require "kong.tools.utils" local Events = require "kong.core.events" local singletons = require "kong.singletons" -local dao_loader = require "kong.tools.dao_loader" -local config_loader = require "kong.tools.config_loader" +local DAOFactory = require "kong.dao.factory" +local conf_loader = require "kong.conf_loader" local plugins_iterator = require "kong.core.plugins_iterator" local ipairs = ipairs -local table_insert = table.insert -local table_sort = table.sort --- Attach a hooks table to the event bus local function attach_hooks(events, hooks) for k, v in pairs(hooks) do events:subscribe(k, v) end end --- Load enabled plugins on the node. --- Get plugins in the DB (distinct by `name`), compare them with plugins --- in `configuration.plugins`. If both lists match, return a list --- of plugins sorted by execution priority for lua-nginx-module's context handlers. --- @treturn table Array of plugins to execute in context handlers. -local function load_node_plugins(configuration) +local function load_plugins(kong_config, events) + local constants = require "kong.constants" + local pl_tablex = require "pl.tablex" + + -- short-lived DAO just to retrieve plugins + local dao = DAOFactory(kong_config) + + local in_db_plugins, sorted_plugins = {}, {} + local plugins = pl_tablex.merge(constants.PLUGINS_AVAILABLE, + kong_config.custom_plugins, true) + ngx.log(ngx.DEBUG, "Discovering used plugins") - local rows, err = singletons.dao.plugins:find_all() - if err then - error(err) - end - local m = {} - for _, row in ipairs(rows) do - m[row.name] = true - end + local rows, err_t = dao.plugins:find_all() + if not rows then return nil, tostring(err_t) end - local distinct_plugins = {} - for plugin_name in pairs(m) do - distinct_plugins[#distinct_plugins + 1] = plugin_name - end + for _, row in ipairs(rows) do in_db_plugins[row.name] = true end - -- Checking that the plugins in the DB are also enabled - for _, v in ipairs(distinct_plugins) do - if not utils.table_contains(configuration.plugins, v) then - error("You are using a plugin that has not been enabled in the configuration: "..v) + -- check all plugins in DB are enabled/installed + for plugin in pairs(in_db_plugins) do + if not plugins[plugin] then + return nil, plugin.." plugin is in use but not enabled" end end - local sorted_plugins = {} - - for _, v in ipairs(configuration.plugins) do - local loaded, plugin_handler_mod = utils.load_module_if_exists("kong.plugins."..v..".handler") - if not loaded then - error("The following plugin has been enabled in the configuration but it is not installed on the system: "..v) - else - local loaded, plugin_schema_mod = utils.load_module_if_exists("kong.plugins."..v..".schema") - if not loaded then - error("Cannot find the schema for the following plugin: "..v) - end - ngx.log(ngx.DEBUG, "Loading plugin: "..v) - table_insert(sorted_plugins, { - name = v, - handler = plugin_handler_mod(), - schema = plugin_schema_mod - }) + -- load installed plugins + for plugin in pairs(plugins) do + local ok, handler = utils.load_module_if_exists("kong.plugins."..plugin..".handler") + if not ok then + return nil, plugin.." plugin is enabled but not installed" + end + + local ok, schema = utils.load_module_if_exists("kong.plugins."..plugin..".schema") + if not ok then + return nil, "no configuration schema found for plugin: "..plugin end + ngx.log(ngx.DEBUG, "Loading plugin: "..plugin) + + sorted_plugins[#sorted_plugins+1] = { + name = plugin, + handler = handler(), + schema = schema + } + -- Attaching hooks - local loaded, plugin_hooks = utils.load_module_if_exists("kong.plugins."..v..".hooks") - if loaded then - attach_hooks(singletons.events, plugin_hooks) + local ok, hooks = utils.load_module_if_exists("kong.plugins."..plugin..".hooks") + if ok then + attach_hooks(events, hooks) end end - table_sort(sorted_plugins, function(a, b) + -- sort plugins by order of execution + table.sort(sorted_plugins, function(a, b) local priority_a = a.handler.PRIORITY or 0 local priority_b = b.handler.PRIORITY or 0 return priority_a > priority_b end) - if configuration.send_anonymous_reports then - table_insert(sorted_plugins, 1, { + -- add reports plugin if not disabled + if kong_config.anonymous_reports then + local reports = require "kong.core.reports" + reports.enable() + sorted_plugins[#sorted_plugins+1] = { name = "reports", - handler = require("kong.core.reports") - }) + handler = reports + } end - return sorted_plugins + -- sorted for handles, name=true for DAO + return {sorted = sorted_plugins, names = plugins} end -- Kong public context handlers. @@ -127,40 +126,28 @@ end local Kong = {} --- Init Kong's environment in the Nginx master process. --- To be called by the lua-nginx-module `init_by_lua` directive. --- Execution: --- - load the configuration from the path computed by the CLI --- - instanciate the DAO Factory --- - load the used plugins --- - load all plugins if used and installed --- - sort the plugins by priority --- --- If any error happens during the initialization of the DAO or plugins, --- it return an nginx error and exit. function Kong.init() - local status, err = pcall(function() - singletons.configuration = config_loader.load(os.getenv("KONG_CONF")) - singletons.events = Events() - singletons.dao = dao_loader.load(singletons.configuration, singletons.events) - singletons.loaded_plugins = load_node_plugins(singletons.configuration) - singletons.serf = Serf(singletons.configuration) - - -- Attach core hooks - attach_hooks(singletons.events, require("kong.core.hooks")) - - if singletons.configuration.send_anonymous_reports then - -- Generate the unique_str inside the module - local reports = require "kong.core.reports" - reports.enable() - end + local pl_path = require "pl.path" - ngx.update_time() - end) - if not status then - ngx.log(ngx.ERR, "Startup error: "..err) - os.exit(1) - end + -- retrieve kong_config + local conf_path = pl_path.join(ngx.config.prefix(), "kong.conf") + local config = assert(conf_loader(conf_path)) + + -- retrieve node plugins + local events = Events() + local plugins = assert(load_plugins(config, events)) + + -- instanciate long-lived DAO + local dao = DAOFactory(config, plugins.names, events) + + -- populate singletons + singletons.loaded_plugins = plugins.sorted + singletons.serf = Serf.new(config, dao) + singletons.dao = dao + singletons.events = events + singletons.configuration = config + + attach_hooks(events, require "kong.core.hooks") end function Kong.init_worker() diff --git a/kong/serf.lua b/kong/serf.lua new file mode 100644 index 00000000000..f5071f79e1c --- /dev/null +++ b/kong/serf.lua @@ -0,0 +1,136 @@ +-- from the previous services.serf module, simply decoupled from +-- the Serf agent supervision logic. + +local pl_stringx = require "pl.stringx" +local pl_utils = require "pl.utils" +local cjson = require "cjson.safe" +local fmt = string.format +local hostname + +do + local pl_utils = require "pl.utils" + local ok, _, stdout, stderr = pl_utils.executeex "/bin/hostname" + if not ok then error(stderr) end + hostname = pl_stringx.strip(stdout) +end + +local Serf = {} +Serf.__index = Serf + +Serf.args_mt = { + __tostring = function(t) + local buf = {} + for k, v in pairs(t) do buf[#buf+1] = k.." '"..v.."'" end + return table.concat(buf, " ") + end +} + +function Serf.new(kong_config, dao) + return setmetatable({ + node_name = hostname..kong_config.cluster_listen, + config = kong_config, + dao = dao + }, Serf) +end + +-- WARN: BAD, this is **blocking** IO. Legacy code from previous Serf +-- implementation that needs to be upgraded. +function Serf:invoke_signal(signal, args, no_rpc) + args = args or {} + setmetatable(args, Serf.args_mt) + local rpc = no_rpc and "" or "-rpc-addr="..self.config.cluster_listen_rpc + local cmd = fmt("serf %s %s %s", signal, rpc, tostring(args)) + + local ok, code, stdout = pl_utils.executeex(cmd) + if not ok or code ~= 0 then return nil, stdout end + + return stdout +end + +function Serf:join_node(address) + return select(2, self:invoke_signal("join", {address})) == nil +end + +function Serf:members() + local res, err = self:invoke_signal("members", {["-format"] = "json"}) + if not res then return nil, err end + + local json, err = cjson.decode(res) + if not json then return nil, err end + + return json.members +end + +function Serf:autojoin() + -- Delete current node just in case it was there + -- (due to an inconsistency caused by a crash) + local _, err = self.dao.nodes:delete {name = self.node_name} + if err then return nil, tostring(err) end + + local nodes, err = self.dao.nodes:find_all() + if err then return nil, tostring(err) + elseif #nodes == 0 then + --logger:warn("Cannot auto-join the cluster because no nodes were found") + else + -- Sort by newest to oldest (although by TTL would be a better sort) + table.sort(nodes, function(a, b) return a.created_at > b.created_at end) + + local joined + for _, v in ipairs(nodes) do + if self:join_node(v.cluster_listening_address) then + --logger:info("Successfully auto-joined "..v.cluster_listening_address) + joined = true + break + else + --logger:warn("Cannot join "..v.cluster_listening_address..". If the node does not exist anymore it will be automatically purged.") + end + end + if not joined then + --logger:warn("Could not join the existing cluster") + end + end + + return true +end + +function Serf:add_node() + local members, err = self:members() + if not members then return nil, err end + + local addr + for _, member in ipairs(members) do + if member.name == self.node_name then + addr = member.addr + break + end + end + + if not addr then + return nil, "can't find current member address" + end + + local _, err = self.dao.nodes:insert({ + name = self.node_name, + cluster_listening_address = pl_stringx.strip(addr) + }, {ttl = 3600}) + if err then return nil, tostring(err) end + + return true +end + +function Serf:event(t_payload) + local payload, err = cjson.encode(t_payload) + if not payload then return nil, err end + + if #payload > 512 then + -- Serf can't send a payload greater than 512 bytes + return nil, "Encoded payload is "..#payload.." and exceeds the limit of 512 bytes!" + end + + return self:invoke_signal("event -coalesce=false kong", { + "'"..payload.."'", + "&" + }) +end + +return Serf diff --git a/kong/templates/kong_defaults.lua b/kong/templates/kong_defaults.lua index 2ede1aca52e..681cacebded 100644 --- a/kong/templates/kong_defaults.lua +++ b/kong/templates/kong_defaults.lua @@ -2,19 +2,18 @@ return [[ admin_listen = 0.0.0.0:8001 proxy_listen = 0.0.0.0:8000 -ssl = on +ssl = off ssl_cert = NONE ssl_cert_key = NONE proxy_listen_ssl = 0.0.0.0:8443 +custom_plugins = NONE + cluster_listen = 0.0.0.0:7946 cluster_listen_rpc = 127.0.0.1:7373 cluster_advertise = NONE cluster_secret = NONE -dnsmasq = on -dns_resolver = 127.0.0.1:8053 - database = postgres pg_host = 127.0.0.1 @@ -23,7 +22,8 @@ pg_user = kong pg_password = kong pg_database = kong -cassandra_contact_points = 127.0.0.1:9042 +cassandra_contact_points = 127.0.0.1 +cassandra_port = 9042 cassandra_keyspace = kong cassandra_repl_strategy = SimpleStrategy cassandra_repl_factor = 1 @@ -36,6 +36,9 @@ cassandra_ssl_trusted_cert = NONE cassandra_username = kong cassandra_password = kong +dnsmasq = on +dns_resolver = 127.0.0.1:8053 + anonymous_reports = on mem_cache_size = 128m diff --git a/kong/templates/nginx.lua b/kong/templates/nginx.lua index 1a84fcc01e4..dd43871ad62 100644 --- a/kong/templates/nginx.lua +++ b/kong/templates/nginx.lua @@ -2,6 +2,8 @@ return [[ worker_processes ${{NGINX_WORKER_PROCESSES}}; daemon ${{NGINX_DAEMON}}; +error_log logs/error.log debug; + events { # if nginx_optimizations then worker_connections ${{ULIMIT}}; diff --git a/kong/templates/nginx_kong.lua b/kong/templates/nginx_kong.lua index 8f8d8cf6698..08f2c153f8a 100644 --- a/kong/templates/nginx_kong.lua +++ b/kong/templates/nginx_kong.lua @@ -2,7 +2,7 @@ return [[ resolver ${{DNS_RESOLVER}} ipv6=off; charset UTF-8; -error_log logs/error.log error; +error_log logs/error.log debug; access_log logs/access.log; # if nginx_optimizations then diff --git a/kong/tools/dao_loader.lua b/kong/tools/dao_loader.lua index da3c05c3b33..2924544c8dd 100644 --- a/kong/tools/dao_loader.lua +++ b/kong/tools/dao_loader.lua @@ -2,8 +2,8 @@ local Factory = require "kong.dao.factory" local _M = {} -function _M.load(config, events_handler) - return Factory(config.database, config.dao_config, config.plugins, events_handler) +function _M.load(kong_config, events_handler) + return Factory(kong_config, kong_config.plugins, events_handler) end return _M diff --git a/spec/unit/99-conf/01-conf_loader_spec.lua b/spec/unit/99-conf/01-conf_loader_spec.lua index 2a7d5884d97..4a2c2bf6097 100644 --- a/spec/unit/99-conf/01-conf_loader_spec.lua +++ b/spec/unit/99-conf/01-conf_loader_spec.lua @@ -9,8 +9,8 @@ describe("Configuration loader", function() assert.equal("0.0.0.0:8001", conf.admin_listen) assert.equal("0.0.0.0:8000", conf.proxy_listen) assert.equal("0.0.0.0:8443", conf.proxy_listen_ssl) - assert.equal("", conf.ssl_cert) -- check placeholder value - assert.equal("", conf.ssl_cert_key) + assert.is_nil(conf.ssl_cert) -- check placeholder value + assert.is_nil(conf.ssl_cert_key) assert.is_nil(getmetatable(conf)) end) it("loads a given file, with higher precedence", function() @@ -48,6 +48,15 @@ describe("Configuration loader", function() })) assert.is_nil(conf.stub_property) end) + it("loads custom plugins", function() + local conf = assert(conf_loader()) + assert.same({}, conf.custom_plugins) + + conf = assert(conf_loader(nil, { + custom_plugins = "hello-world,my-plugin" + })) + assert.same({"hello-world", "my-plugin"}, conf.custom_plugins) + end) describe("inferences", function() it("infer booleans (on/off/true/false strings)", function() @@ -130,5 +139,19 @@ describe("Configuration loader", function() assert.equal("no file at: inexistent", err) assert.is_nil(conf) end) + it("requires cert and key if SSL is enabled", function() + local conf, err = conf_loader(nil, { + ssl = true + }) + assert.equal("ssl_cert required if SSL enabled", err) + assert.is_nil(conf) + + conf, err = conf_loader(nil, { + ssl = true, + ssl_cert = "/path/cert.pem" + }) + assert.equal("ssl_cert_key required if SSL enabled", err) + assert.is_nil(conf) + end) end) end) diff --git a/spec/unit/99-conf/02-conf_compilation_spec.lua b/spec/unit/99-conf/02-conf_compilation_spec.lua index b4808756cb0..e6a0ee17c5d 100644 --- a/spec/unit/99-conf/02-conf_compilation_spec.lua +++ b/spec/unit/99-conf/02-conf_compilation_spec.lua @@ -114,9 +114,6 @@ describe("NGINX conf compiler", function() assert.truthy(exists(join(prefix, "logs", "access.log"))) end) it("creates Kong conf", function() - local pl_config = require "pl.config" - local pl_stringio = require "pl.stringio" - assert(nginx_conf_compiler.prepare_prefix(helpers.test_conf, prefix)) local path = assert.truthy(exists(join(prefix, "kong.conf")))