Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
feat(http-log) adds log buffering to http-log plugin
Adds three new properties to the configuration of http-log: * `retry_count` - default 10 * `queue_size` - default 1 * `flush_timeout` - default 2 Their behaviors are the same as the identically-named options in the `galileo` plugin. The default of `queue_size` is set to 1 so that the default behavior of the plugin remains backwards-compatible. When `queue_size` is 1, it makes one HTTP request per logged entry, containing a single JSON object in each. When `queue_size` is greater than 1, the HTTP connection will send a JSON array containing one or more JSON objects in it, up to the configured queue size. The new fields are not required, and when absent their defaults mean that they behave the same as the http-log plugin always did, meaning this requires no migrations. Includes a test case using the mock log server.
- Loading branch information
Showing
7 changed files
with
341 additions
and
99 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,114 @@ | ||
local url = require "socket.url" | ||
local http = require "resty.http" | ||
|
||
|
||
local sender = {} | ||
|
||
|
||
local ngx_encode_base64 = ngx.encode_base64 | ||
local ERR = ngx.ERR | ||
|
||
|
||
-- Parse host url. | ||
-- @param `url` host url | ||
-- @return `parsed_url` a table with host details like domain name, port, path etc | ||
local function parse_url(host_url) | ||
local parsed_url = url.parse(host_url) | ||
if not parsed_url.port then | ||
if parsed_url.scheme == "http" then | ||
parsed_url.port = 80 | ||
elseif parsed_url.scheme == "https" then | ||
parsed_url.port = 443 | ||
end | ||
end | ||
if not parsed_url.path then | ||
parsed_url.path = "/" | ||
end | ||
return parsed_url | ||
end | ||
|
||
|
||
-- Log to a Http end point. | ||
-- @param `bodies` raw http bodies to be logged | ||
local function send(self, bodies) | ||
local log = self.log | ||
|
||
local ok, err | ||
local parsed_url = parse_url(self.http_endpoint) | ||
local host = parsed_url.host | ||
local port = tonumber(parsed_url.port) | ||
|
||
if type(bodies) == "string" then | ||
bodies = { bodies } | ||
end | ||
for _, body in ipairs(bodies) do | ||
local httpc = http.new() | ||
httpc:set_timeout(self.timeout) | ||
ok, err = httpc:connect(host, port) | ||
if not ok then | ||
log(ERR, "failed to connect to ", host, ":", tostring(port), ": ", err) | ||
return false | ||
end | ||
|
||
if parsed_url.scheme == "https" then | ||
local _, err = httpc:ssl_handshake(true, host, false) | ||
if err then | ||
log(ERR, "failed to do SSL handshake with ", | ||
host, ":", tostring(port), ": ", err) | ||
return false | ||
end | ||
end | ||
|
||
local res, err = httpc:request({ | ||
method = self.method, | ||
path = parsed_url.path, | ||
query = parsed_url.query, | ||
headers = { | ||
["Host"] = parsed_url.host, | ||
["Content-Type"] = self.content_type, | ||
["Content-Length"] = #body, | ||
["Authorization"] = parsed_url.userinfo and ( | ||
"Basic " .. ngx_encode_base64(parsed_url.userinfo) | ||
), | ||
}, | ||
body = body, | ||
}) | ||
if not res then | ||
log(ERR, "failed request to ", host, ":", tostring(port), ": ", err) | ||
end | ||
|
||
-- read and discard body | ||
-- TODO should we fail if response status was >= 500 ? | ||
res:read_body() | ||
|
||
ok, err = httpc:set_keepalive(self.keepalive) | ||
if not ok then | ||
log(ERR, "failed keepalive for ", host, ":", tostring(port), ": ", err) | ||
end | ||
end | ||
|
||
return true | ||
end | ||
|
||
|
||
function sender.new(conf, log) | ||
if type(log) ~= "function" then | ||
error("arg #2 (log) must be a function") | ||
end | ||
|
||
local self = { | ||
http_endpoint = conf.http_endpoint, | ||
content_type = conf.content_type, | ||
keepalive = conf.keepalive, | ||
timeout = conf.timeout, | ||
method = conf.method, | ||
|
||
send = send, | ||
log = log, | ||
} | ||
|
||
return self | ||
end | ||
|
||
|
||
return sender |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
-- JSON array producer object for using the Generic Logging Buffer. | ||
local cjson = require("cjson") | ||
|
||
|
||
local json_producer = {} | ||
|
||
|
||
local cjson_encode = cjson.encode | ||
|
||
|
||
local function add_entry(self, data) | ||
if not self.encoded then | ||
data = cjson_encode(data) | ||
end | ||
local n = #self.output | ||
if n == 0 then | ||
self.output[1] = "[" | ||
else | ||
self.output[n+1] = "," | ||
end | ||
self.output[n+2] = data | ||
self.bytes = self.bytes + #data + 1 | ||
return true, (n + 2) / 2 | ||
end | ||
|
||
|
||
local function produce(self) | ||
local count = #self.output / 2 | ||
self.output[#self.output + 1] = "]" | ||
local data = table.concat(self.output) | ||
return data, count, #data | ||
end | ||
|
||
|
||
local function reset(self) | ||
self.output = {} | ||
self.bytes = 1 | ||
end | ||
|
||
|
||
-- Produces the given entries into a JSON array. | ||
-- @param raw_tree (boolean) | ||
-- If `encoded` is `true`, entries are assumed to be strings | ||
-- that already represent JSON-encoded data. | ||
-- If `encoded` is `false`, entries are assumed to be Lua objects | ||
-- that need to be encoded during serialization. | ||
function json_producer.new(encoded) | ||
if encoded ~= nil and type(encoded) ~= "boolean" then | ||
error("arg 2 (encoded) must be boolean") | ||
end | ||
|
||
local self = { | ||
output = {}, | ||
bytes = 1, | ||
encoded = encoded, | ||
|
||
add_entry = add_entry, | ||
produce = produce, | ||
reset = reset, | ||
} | ||
|
||
return self | ||
end | ||
|
||
|
||
return json_producer |
Oops, something went wrong.
de4a002
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@hishamhm I decided to test some of this logic on one of our own branches with tweaks for 0.14.1. I found that a change between the IMPL as it stands vs older logic is the reliance on a route.id or api.id(for deprecated entities). I think the appropriate behavior would be if route or api id is nil to use a static buffer key and let that batch for all proxy calls that are not necessarily associated with a route/api, because even 404 not founds to the gateway should get logged. Plugin will throw a nil pointer exception in current state because buffer[nil] on gateway paths that don't exist.
https://github.com/Kong/kong/blob/next/kong/plugins/http-log/handler.lua#L51
I made mine something like
But I also want to thank you for this revised logic, the batching plus leveraging the rest client lib seems to have resolved problems we had with the older plugin(lots of ssl and tcp connection problems) 👍 .