Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(prometheus) add subsystem label to upstream's health info #7802

Merged
merged 1 commit into from Sep 3, 2021
Merged

fix(prometheus) add subsystem label to upstream's health info #7802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 5 additions & 5 deletions kong/plugins/prometheus/exporter.lua
View file
Open in desktop
Expand Up @@ -60,7 +60,7 @@ local function init()
"Health status of targets of upstream. " ..
"States = healthchecks_off|healthy|unhealthy|dns_error, " ..
"value is 1 when state is populated.",
{"upstream", "target", "address", "state"})
{"upstream", "target", "address", "state", "subsystem"})

local memory_stats = {}
memory_stats.worker_vms = prometheus:gauge("memory_workers_lua_vms_bytes",
Expand Down Expand Up @@ -140,10 +140,10 @@ end
local labels_table = {0, 0, 0}
local labels_table4 = {0, 0, 0, 0}
local upstream_target_addr_health_table = {
{ value = 0, labels = { 0, 0, 0, "healthchecks_off" } },
{ value = 0, labels = { 0, 0, 0, "healthy" } },
{ value = 0, labels = { 0, 0, 0, "unhealthy" } },
{ value = 0, labels = { 0, 0, 0, "dns_error" } },
{ value = 0, labels = { 0, 0, 0, "healthchecks_off", ngx.config.subsystem } },
{ value = 0, labels = { 0, 0, 0, "healthy", ngx.config.subsystem } },
{ value = 0, labels = { 0, 0, 0, "unhealthy", ngx.config.subsystem } },
{ value = 0, labels = { 0, 0, 0, "dns_error", ngx.config.subsystem } },
}

local function set_healthiness_metrics(table, upstream, target, address, status, metrics_bucket)
Expand Down
2 changes: 1 addition & 1 deletion kong/plugins/prometheus/handler.lua
View file
Open in desktop
Expand Up @@ -7,7 +7,7 @@ prometheus.init()

local PrometheusHandler = {
PRIORITY = 13,
VERSION = "1.3.0",
VERSION = "1.4.0",
}

function PrometheusHandler.init_worker()
Expand Down
51 changes: 35 additions & 16 deletions spec/03-plugins/26-prometheus/04-status_api_spec.lua
View file
Open in desktop
Expand Up @@ -295,11 +295,11 @@ describe("Plugin: prometheus (access via status API)", function()
})

body = assert.res_status(200, res)
return body:find('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthchecks_off"} 1', nil, true)
return body:find('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthchecks_off",subsystem="http"} 1', nil, true)
end)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="unhealthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="dns_error"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="unhealthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream-healthchecksoff",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="dns_error",subsystem="http"} 0', body, nil, true)
end)

it("exposes upstream's target health metrics - healthy", function()
Expand All @@ -311,11 +311,11 @@ describe("Plugin: prometheus (access via status API)", function()
})

body = assert.res_status(200, res)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthy"} 1', nil, true)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthy",subsystem="http"} 1', nil, true)
end)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthchecks_off"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="unhealthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="dns_error"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="healthchecks_off",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="unhealthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",address="' .. helpers.mock_upstream_host .. ':' .. helpers.mock_upstream_port .. '",state="dns_error",subsystem="http"} 0', body, nil, true)
end)

it("exposes upstream's target health metrics - unhealthy", function()
Expand All @@ -327,11 +327,11 @@ describe("Plugin: prometheus (access via status API)", function()
})

body = assert.res_status(200, res)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="unhealthy"} 1', nil, true)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="unhealthy",subsystem="http"} 1', nil, true)
end)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="healthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="healthchecks_off"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="dns_error"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="healthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="healthchecks_off",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="' .. helpers.mock_upstream_host .. ':8001",address="' .. helpers.mock_upstream_host .. ':8001",state="dns_error",subsystem="http"} 0', body, nil, true)
end)

it("exposes upstream's target health metrics - dns_error", function()
Expand All @@ -343,11 +343,30 @@ describe("Plugin: prometheus (access via status API)", function()
})

body = assert.res_status(200, res)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="dns_error"} 1', nil, true)
return body:find('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="dns_error",subsystem="http"} 1', nil, true)
end)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="unhealthy"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthchecks_off"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="unhealthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthchecks_off",subsystem="http"} 0', body, nil, true)
end)

it("adds subsystem label to upstream's target health metrics", function()
local body
helpers.wait_until(function()
local res = assert(status_client:send {
method = "GET",
path = "/metrics",
})

body = assert.res_status(200, res)
return body:gmatch('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="dns_error",subsystem="%w+"} 1', nil, true)
end)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthy",subsystem="stream"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="unhealthy",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="unhealthy",subsystem="stream"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthchecks_off",subsystem="http"} 0', body, nil, true)
assert.matches('kong_upstream_target_health{upstream="mock-upstream",target="some-random-dns:80",address="",state="healthchecks_off",subsystem="stream"} 0', body, nil, true)
end)

it("remove metrics from deleted upstreams", function()
Expand Down