Skip to content

Commit

Permalink
feat(prometheus) add subsystem label to upstream's health info (#7802)
Browse files Browse the repository at this point in the history
  • Loading branch information
@gszr
gszr committed Sep 3, 2021
1 parent e86c50a commit 8e4c003
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 22 deletions.
10 changes: 5 additions & 5 deletions kong/plugins/prometheus/exporter.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ local function init()
"Health status of targets of upstream. " ..
"States = healthchecks_off|healthy|unhealthy|dns_error, " ..
"value is 1 when state is populated.",
{"upstream", "target", "address", "state"})
{"upstream", "target", "address", "state", "subsystem"})

local memory_stats = {}
memory_stats.worker_vms = prometheus:gauge("memory_workers_lua_vms_bytes",
Expand Down Expand Up @@ -140,10 +140,10 @@ end
local labels_table = {0, 0, 0}
local labels_table4 = {0, 0, 0, 0}
local upstream_target_addr_health_table = {
{ value = 0, labels = { 0, 0, 0, "healthchecks_off" } },
{ value = 0, labels = { 0, 0, 0, "healthy" } },
{ value = 0, labels = { 0, 0, 0, "unhealthy" } },
{ value = 0, labels = { 0, 0, 0, "dns_error" } },
{ value = 0, labels = { 0, 0, 0, "healthchecks_off", ngx.config.subsystem } },
{ value = 0, labels = { 0, 0, 0, "healthy", ngx.config.subsystem } },
{ value = 0, labels = { 0, 0, 0, "unhealthy", ngx.config.subsystem } },
{ value = 0, labels = { 0, 0, 0, "dns_error", ngx.config.subsystem } },
}

local function set_healthiness_metrics(table, upstream, target, address, status, metrics_bucket)
Expand Down
2 changes: 1 addition & 1 deletion kong/plugins/prometheus/handler.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ prometheus.init()

local PrometheusHandler = {
PRIORITY = 13,
VERSION = "1.3.0",
VERSION = "1.4.0",
}

function PrometheusHandler.init_worker()
Expand Down
51 changes: 35 additions & 16 deletions spec/03-plugins/26-prometheus/04-status_api_spec.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,11 +295,11 @@ describe("Plugin: prometheus (access via status API)", function()
})

body = assert.res_status(200, res)
return body:find('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthchecks_off"} 1', nil, true)
return body:find('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthchecks_off",subsystem="http"} 1', nil, true)
end)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="unhealthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="dns_error"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="unhealthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="dns_error",subsystem="http"} 0', body, nil, true)
end)

it("exposes upstream's target health metrics - healthy", function()
Expand All @@ -311,11 +311,11 @@ describe("Plugin: prometheus (access via status API)", function()
})

body = assert.res_status(200, res)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthy"} 1', nil, true)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthy",subsystem="http"} 1', nil, true)
end)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthchecks_off"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="unhealthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="dns_error"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthchecks_off",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="unhealthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="dns_error",subsystem="http"} 0', body, nil, true)
end)

it("exposes upstream's target health metrics - unhealthy", function()
Expand All @@ -327,11 +327,11 @@ describe("Plugin: prometheus (access via status API)", function()
})

body = assert.res_status(200, res)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="unhealthy"} 1', nil, true)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="unhealthy",subsystem="http"} 1', nil, true)
end)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="healthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="healthchecks_off"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="dns_error"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="healthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="healthchecks_off",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="dns_error",subsystem="http"} 0', body, nil, true)
end)

it("exposes upstream's target health metrics - dns_error", function()
Expand All @@ -343,11 +343,30 @@ describe("Plugin: prometheus (access via status API)", function()
})

body = assert.res_status(200, res)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="dns_error"} 1', nil, true)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="dns_error",subsystem="http"} 1', nil, true)
end)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="unhealthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthchecks_off"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="unhealthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthchecks_off",subsystem="http"} 0', body, nil, true)
end)

it("adds subsystem label to upstream's target health metrics", function()
local body
helpers.wait_until(function()
local res = assert(status_client:send {
method = "GET",
path = "/metrics",
})

body = assert.res_status(200, res)
return body:gmatch('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="dns_error",subsystem="%w+"} 1', nil, true)
end)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthy",subsystem="stream"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="unhealthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="unhealthy",subsystem="stream"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthchecks_off",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthchecks_off",subsystem="stream"} 0', body, nil, true)
end)

it("remove metrics from deleted upstreams", function()
Expand Down

0 comments on commit 8e4c003

Please sign in to comment.