feat(plugins) new Galileo plugin

kong-0.8.1-0.rockspec

@@ -14,6 +14,7 @@ dependencies = {
   "luasec ~> 0.5-2",
 
   "penlight ~> 1.3.2",
+  "lua-resty-http ~> 0.07-0",
   "lua_uuid ~> 0.2.0-2",
   "lua_system_constants ~> 0.1.1-0",
   "luatz ~> 0.3-1",
@@ -44,7 +45,6 @@ build = {
 
     ["classic"] = "kong/vendor/classic.lua",
     ["lapp"] = "kong/vendor/lapp.lua",
-    ["resty_http"] = "kong/vendor/resty_http.lua",
 
     ["kong.meta"] = "kong/meta.lua",
     ["kong.constants"] = "kong/constants.lua",
@@ -149,7 +149,6 @@ build = {
     ["kong.plugins.oauth2.api"] = "kong/plugins/oauth2/api.lua",
 
     ["kong.plugins.log-serializers.basic"] = "kong/plugins/log-serializers/basic.lua",
-    ["kong.plugins.log-serializers.alf"] = "kong/plugins/log-serializers/alf.lua",
     ["kong.plugins.log-serializers.runscope"] = "kong/plugins/log-serializers/runscope.lua",
 
     ["kong.plugins.tcp-log.handler"] = "kong/plugins/tcp-log/handler.lua",
@@ -168,10 +167,11 @@ build = {
     ["kong.plugins.runscope.schema"] = "kong/plugins/runscope/schema.lua",
     ["kong.plugins.runscope.log"] = "kong/plugins/runscope/log.lua",
 
-    ["kong.plugins.mashape-analytics.migrations.cassandra"] = "kong/plugins/mashape-analytics/migrations/cassandra.lua",
-    ["kong.plugins.mashape-analytics.handler"] = "kong/plugins/mashape-analytics/handler.lua",
-    ["kong.plugins.mashape-analytics.schema"] = "kong/plugins/mashape-analytics/schema.lua",
-    ["kong.plugins.mashape-analytics.buffer"] = "kong/plugins/mashape-analytics/buffer.lua",
+    ["kong.plugins.galileo.migrations.cassandra"] = "kong/plugins/galileo/migrations/cassandra.lua",
+    ["kong.plugins.galileo.handler"] = "kong/plugins/galileo/handler.lua",
+    ["kong.plugins.galileo.schema"] = "kong/plugins/galileo/schema.lua",
+    ["kong.plugins.galileo.buffer"] = "kong/plugins/galileo/buffer.lua",
+    ["kong.plugins.galileo.alf"] = "kong/plugins/galileo/alf.lua",
 
     ["kong.plugins.rate-limiting.migrations.cassandra"] = "kong/plugins/rate-limiting/migrations/cassandra.lua",
     ["kong.plugins.rate-limiting.migrations.postgres"] = "kong/plugins/rate-limiting/migrations/postgres.lua",

kong/constants.lua

@@ -11,7 +11,7 @@ return {
   PLUGINS_AVAILABLE = {
     "ssl", "jwt", "acl", "correlation-id", "cors", "oauth2", "tcp-log", "udp-log", "file-log",
     "http-log", "key-auth", "hmac-auth", "basic-auth", "ip-restriction",
-    "mashape-analytics", "request-transformer", "response-transformer",
+    "galileo", "request-transformer", "response-transformer",
     "request-size-limiting", "rate-limiting", "response-ratelimiting", "syslog",
     "loggly", "datadog", "runscope", "ldap-auth", "statsd"
   },

kong/plugins/galileo/alf.lua

@@ -0,0 +1,273 @@
+-- ALF implementation for ngx_lua/Kong
+-- ALF version: 1.1.0
+-- @version 2.0.0
+-- @see https://github.com/Mashape/galileo-agent-spec
+--
+-- Incompatibilities with ALF 1.1 and important notes
+-- ==================================================
+-- * The following fields cannot be retrieved as of ngx_lua 0.10.2:
+--     * response.statusText
+--     * response.headersSize
+--
+-- * Kong can modify the request/response due to its nature, hence,
+--   we distinguish the original req/res from the current req/res
+--     * request.headersSize will be the size of the _original_ headers
+--       received by Kong
+--     * request.headers will contain the _current_ headers
+--     * response.headers will contain the _current_ headers
+--
+-- * bodyCaptured properties are determined using HTTP headers
+-- * timings.blocked is ignored
+-- * timings.connect is ignored
+--
+-- * we are waiting for lua-cjson support of granular 'empty table
+--   as JSON arrays' to be released in OpenResty. Until then, we
+--   have to use a workaround involving string substituion, which
+--   is slower and limits the maximum size of ALFs we can deal with.
+--   ALFs are thus limited to 20MB
+
+local cjson = require "cjson.safe"
+local resp_get_headers = ngx.resp.get_headers
+local req_start_time = ngx.req.start_time
+local req_get_method = ngx.req.get_method
+local req_get_headers = ngx.req.get_headers
+local req_get_uri_args = ngx.req.get_uri_args
+local req_raw_header = ngx.req.raw_header
+local encode_base64 = ngx.encode_base64
+local http_version = ngx.req.http_version
+local setmetatable = setmetatable
+local tonumber = tonumber
+local os_date = os.date
+local type = type
+local pairs = pairs
+
+local _M = {
+  _VERSION = "2.0.0",
+  _ALF_VERSION = "1.1.0",
+  _ALF_CREATOR = "galileo-agent-kong"
+}
+
+local _mt = {
+  __index = _M
+}
+
+function _M.new(log_bodies, server_addr)
+  local alf = {
+    server_addr = server_addr,
+    log_bodies = log_bodies,
+    entries = {}
+  }
+
+  return setmetatable(alf, _mt)
+end
+
+local _empty_arr_placeholder = "__empty_array_placeholder__"
+local _empty_arr_t = {_empty_arr_placeholder}
+
+-- Convert a table such as returned by ngx.*.get_headers()
+-- to integer-indexed arrays.
+-- @warn Encoding of empty arrays workaround forces us
+-- to replace empty arrays with a placeholder to be substituted
+-- at serialization time.
+-- waiting on the releast of:
+-- https://github.com/openresty/lua-cjson/pull/6
+local function hash_to_array(t)
+  local arr = {}
+  for k, v in pairs(t) do
+    if type(v) == "table" then
+      for i = 1, #v do
+        arr[#arr+1] = {name = k, value = v[i]}
+      end
+    else
+      arr[#arr+1] = {name = k, value = v}
+    end
+  end
+
+  if #arr == 0 then
+    return _empty_arr_t
+  else
+    return arr
+  end
+end
+
+local function get_header(t, name, default)
+  local v = t[name]
+  if not v then
+    return default
+  elseif type(v) == "table" then
+    return v[#v]
+  end
+  return v
+end
+
+--- Add an entry to the ALF's `entries`
+-- @param[type=table] _ngx The ngx table, containing .var and .ctx
+-- @param[type=string] req_body_str The request body
+-- @param[type=string] res_body_str The response body
+-- @treturn table The entry created
+-- @treturn number The new size of the `entries` array
+function _M:add_entry(_ngx, req_body_str, resp_body_str)
+  if not self.entries then
+    return nil, "no entries table"
+  elseif not _ngx then
+    return nil, "arg #1 (_ngx) must be given"
+  elseif req_body_str ~= nil and type(req_body_str) ~= "string" then
+    return nil, "arg #2 (req_body_str) must be a string"
+  elseif resp_body_str ~= nil and type(resp_body_str) ~= "string" then
+    return nil, "arg #3 (resp_body_str) must be a string"
+  end
+
+  -- retrieval
+  local var = _ngx.var
+  local ctx = _ngx.ctx
+  local http_version = "HTTP/"..http_version()
+  local request_headers = req_get_headers()
+  local request_content_len = get_header(request_headers, "content-length", 0)
+  local request_transfer_encoding = get_header(request_headers, "transfer-encoding")
+  local request_content_type = get_header(request_headers, "content-type",
+                                          "application/octet-stream")
+
+  -- if log_bodies is false, we don't want to still call
+  -- ngx.req.read_body() anyways, hence we rely on RFC 2616
+  -- to determine if the request seems to have a body.
+  local req_has_body = tonumber(request_content_len) > 0
+                       or request_transfer_encoding ~= nil
+                       or request_content_type == "multipart/byteranges"
+
+  local resp_headers = resp_get_headers()
+  local resp_content_len = get_header(resp_headers, "content-length", 0)
+  local resp_transfer_encoding = get_header(resp_headers, "transfer-encoding")
+  local resp_content_type = get_header(resp_headers, "content-type",
+                            "application/octet-stream")
+
+  local resp_has_body = tonumber(resp_content_len) > 0
+                        or resp_transfer_encoding ~= nil
+                        or resp_content_type == "multipart/byteranges"
+
+  -- request.postData. we don't check has_body here, but rather
+  -- stick to what the request really contains, since it was
+  -- already read anyways.
+  local post_data, response_content
+  local req_body_size, resp_body_size = 0, 0
+
+  if self.log_bodies then
+    if req_body_str then
+      req_body_size = #req_body_str
+      post_data = {
+        text = encode_base64(req_body_str),
+        encoding = "base64",
+        mimeType = request_content_type
+      }
+    end
+
+    if resp_body_str then
+      resp_body_size = #resp_body_str
+      response_content = {
+        text = encode_base64(resp_body_str),
+        encoding = "base64",
+        mimeType = resp_content_type
+      }
+    end
+  end
+
+  -- timings
+  local send_t = ctx.KONG_PROXY_LATENCY or 0
+  local wait_t = ctx.KONG_WAITING_TIME or 0
+  local receive_t = ctx.KONG_RECEIVE_TIME or 0
+
+  local idx = #self.entries + 1
+
+  self.entries[idx] = {
+    time = send_t + wait_t + receive_t,
+    startedDateTime = os_date("!%Y-%m-%dT%TZ", req_start_time()),
+    serverIPAddress = self.server_addr,
+    clientIPAddress = var.remote_addr,
+    request = {
+      httpVersion = http_version,
+      method = req_get_method(),
+      url = var.scheme .. "://" .. var.host .. var.request_uri,
+      queryString = hash_to_array(req_get_uri_args()),
+      headers = hash_to_array(request_headers),
+      headersSize = #req_raw_header(),
+      postData = post_data,
+      bodyCaptured = req_has_body,
+      bodySize = req_body_size,
+    },
+    response = {
+      status = _ngx.status,
+      statusText = "",
+      httpVersion = http_version,
+      headers = hash_to_array(resp_headers),
+      content = response_content,
+      headersSize = 0,
+      bodyCaptured = resp_has_body,
+      bodySize = resp_body_size
+    },
+    timings = {
+      send = send_t,
+      wait = wait_t,
+      receive = receive_t
+    }
+  }
+
+  return self.entries[idx], idx
+end
+
+local buf = {
+   version = _M._ALF_VERSION,
+   serviceToken = nil,
+   environment = nil,
+   har = {
+     log = {
+       creator = {
+         name = _M._ALF_CREATOR,
+         version = _M._VERSION
+       },
+       entries = nil
+     }
+   }
+ }
+
+local gsub = string.gsub
+local pat = '"'.._empty_arr_placeholder..'"'
+local _alf_max_size = 20 * 2^20
+
+--- Encode the current ALF to JSON
+-- @param[type=string] service_token The ALF `serviceToken`
+-- @param[type=string] environment (optional) The ALF `environment`
+-- @treturn string The ALF, JSON encoded
+function _M:serialize(service_token, environment)
+  if not self.entries then
+    return nil, "no entries table"
+  elseif type(service_token) ~= "string" then
+    return nil, "arg #1 (service_token) must be a string"
+  elseif environment ~= nil and type(environment) ~= "string" then
+    return nil, "arg #2 (environment) must be a string"
+  end
+
+  buf.serviceToken = service_token
+  buf.environment = environment
+  buf.har.log.entries = self.entries
+
+  -- tmp workaround for empty arrays
+  -- this prevents us from dealing with ALFs
+  -- larger than a few MBs at once
+  local encoded =  cjson.encode(buf)
+
+  if #encoded > _alf_max_size then
+    return nil, "ALF too large (> 20MB)"
+  end
+
+  encoded = gsub(encoded, pat, "")
+  return gsub(encoded, "\\/", "/"), #self.entries
+
+  --return cjson.encode(buf)
+end
+
+--- Empty the ALF
+function _M:reset()
+  self.entries = {}
+end
+
+return _M
+