Namespace as label and custom metric prefix options (#121)

martin-helmich · May 5, 2020 · 8ffdfa9 · 8ffdfa9
1 parent a554c54
commit 8ffdfa9
Show file tree

Hide file tree

Showing 4 changed files with 266 additions and 41 deletions.
diff --git a/README.adoc b/README.adoc
@@ -236,6 +236,8 @@ These metrics are exported:
 
 Additional labels can be configured in the configuration file (see below).
 
+<namespace> can be ommited or overrided - see <<Namespace-as-labels>>
+
 == Configuration file
 
 You can specify a configuration file to read at startup. The configuration file
@@ -270,6 +272,9 @@ namespace "app1" {
       "/var/log/nginx/app1/access.log"
     ]
   }
+  #metrics_override = { prefix = "myprefix" }
+  #namespace_label = "vhost"
+
   labels {
     app = "application-one"
     environment = "production"
@@ -316,6 +321,9 @@ namespaces:
     source:
       files:
         - /var/log/nginx/app1/access.log
+    #metrics_override:
+    #  prefix: "myprefix"
+    #namespace_label: "vhost"
     labels:
       app: "application-one"
       environment: "production"
@@ -330,6 +338,61 @@ namespaces:
 
 Advanced features
 -----------------
+### Namespace as labels
+
+It's more convinient to have same metrics across different
+namespaces (with different log formats and names).
+
+This can be done in two steps:
+
+1. Override Prometheus metrics namespace to some common prefix (`metrics_override`)
+2. Set label name for nginxlog-exporter's config namespace (`namespace_label`)
+
+[source,hcl]
+----
+namespace "app1" {
+  ...
+  metrics_override = { prefix = "myprefix" }
+  namespace_label = "vhost"
+  ...
+}
+
+namespace "app2" {
+  ...
+  metrics_override = { prefix = "myprefix" }
+  namespace_label = "vhost"
+  ...
+}
+----
+
+Exported metrics will have the following format:
+[source]
+----
+myprefix_http_response_count_total{vhost="app1", ...}
+myprefix_http_response_count_total{vhost="app2", ...}
+...
+----
+
+* `prefix` can be set to `""`, resulting metrics like `http_response_count_total{...}`
+* `namespace_label` can be ommited - so you have full control on metric format
+
+Some details and history: https://github.com/martin-helmich/prometheus-nginxlog-exporter/issues/13[Issue #13]
+
+### Custom labels pass-through
+Partial case of <<Dynamic-re-labeling>>:
+
+[source,hcl]
+----
+namespace "app1" {
+  format = "$remote_addr - $remote_user [$time_local] ... \"$geoip_country_code\" $upstream_addr"
+  ...
+  relabel "upstream_addr" { from = "upstream_addr" }
+  relabel "country" { from = "geoip_country_code" }
+  ...
+}
+----
+
+Exported metrics will have `upstream_addr` and `country` labels.
 
 ### Log sources
 

diff --git a/config/loader_test.go b/config/loader_test.go
@@ -122,6 +122,8 @@ func assertConfigContents(t *testing.T, cfg Config) {
 	assert.Equal(t, []string{"test.log", "foo.log"}, n.SourceFiles)
 	assert.Equal(t, FileSource{"test.log", "foo.log"}, n.SourceData.Files)
 	assert.Equal(t, "magicapp", n.Labels["app"])
+	assert.Nil(t, n.NamespaceLabels)
+	assert.Nil(t, n.MetricsOverride)
 
 	require.Len(t, n.RelabelConfigs, 2)
 	assert.Equal(t, "user", n.RelabelConfigs[0].TargetLabel)
@@ -152,3 +154,120 @@ func TestLoadsYAMLConfigFile(t *testing.T) {
 	assert.Nil(t, err, "unexpected error: %v", err)
 	assertConfigContents(t, cfg)
 }
+
+const HCLLabeledInput = `
+listen {
+  address = "10.0.0.1"
+  port = 4040
+}
+enable_experimental = true
+
+namespace "default" {
+  source_files = [
+    "access.log"
+  ]
+  format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
+
+  metrics_override = { prefix = "nginxlog" }
+}
+
+namespace "myapp1" {
+  source_files = [
+    "myapp1-access.log"
+  ]
+  format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
+
+  metrics_override = { prefix = "" }
+  namespace_label = "vhost"
+}
+
+namespace "myapp2" {
+  source_files = [
+    "myapp2-access.log"
+  ]
+  format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
+
+  metrics_override = { prefix = "" }
+  namespace_label = "vhost"
+}
+`
+
+const YAMLLabeledInput = `
+listen:
+  address: "10.0.0.1"
+  port: 4040
+enable_experimental: true
+
+namespaces:
+  - name: default
+    source_files:
+      - access.log
+    format: "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
+    metrics_override:
+      prefix: "nginxlog"
+
+  - name: myapp1
+    source_files:
+      - myapp1-access.log
+    format: "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
+    metrics_override:
+      prefix: ""
+    namespace_label: "vhost"
+
+  - name: myapp2
+    source_files:
+      - myapp2-access.log
+    format: "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
+    metrics_override:
+      prefix: ""
+    namespace_label: "vhost"
+`
+
+func assertLabeledConfigContents(t *testing.T, cfg Config) {
+	assert.Equal(t, "10.0.0.1", cfg.Listen.Address)
+	assert.Equal(t, 4040, cfg.Listen.Port)
+
+	require.Len(t, cfg.Namespaces, 3)
+
+	for i, n := range cfg.Namespaces {
+		err := n.Compile()
+		assert.Nil(t, err, "unexpected error: %v", err)
+
+		switch i {
+		case 0:
+			assert.Equal(t, "default", n.Name)
+			assert.Equal(t, "nginxlog", n.NamespacePrefix)
+			assert.Nil(t, n.NamespaceLabels)
+		case 1:
+			assert.Equal(t, "myapp1", n.Name)
+			assert.Equal(t, "", n.NamespacePrefix)
+			assert.Equal(t, map[string]string{"vhost": "myapp1"}, n.NamespaceLabels)
+		case 2:
+			assert.Equal(t, "myapp2", n.Name)
+			assert.Equal(t, "", n.NamespacePrefix)
+			assert.Equal(t, map[string]string{"vhost": "myapp2"}, n.NamespaceLabels)
+		}
+	}
+}
+
+func TestLoadsNSLabeledHCLConfigFile(t *testing.T) {
+	t.Parallel()
+
+	buf := bytes.NewBufferString(HCLLabeledInput)
+	cfg := Config{}
+
+	err := LoadConfigFromStream(&cfg, buf, TypeHCL)
+	assert.Nil(t, err, "unexpected error: %v", err)
+	assertLabeledConfigContents(t, cfg)
+}
+
+func TestLoadsNSLabeledYAMLConfigFile(t *testing.T) {
+	t.Parallel()
+
+	buf := bytes.NewBufferString(YAMLLabeledInput)
+	cfg := Config{}
+
+	err := LoadConfigFromStream(&cfg, buf, TypeYAML)
+	assert.Nil(t, err, "unexpected error: %v", err)
+	assertLabeledConfigContents(t, cfg)
+}
diff --git a/config/struct_namespace.go b/config/struct_namespace.go
@@ -7,7 +7,16 @@ import (
 
 // NamespaceConfig is a struct describing single metric namespaces
 type NamespaceConfig struct {
-	Name             string            `hcl:",key"`
+	Name string `hcl:",key"`
+
+	NamespaceLabelName string `hcl:"namespace_label" yaml:"namespace_label"`
+	NamespaceLabels    map[string]string
+
+	MetricsOverride *struct {
+		Prefix string `hcl:"prefix" yaml:"prefix"`
+	} `hcl:"metrics_override" yaml:"metrics_override"`
+	NamespacePrefix string
+
 	SourceFiles      []string          `hcl:"source_files" yaml:"source_files"`
 	SourceData       SourceData        `hcl:"source" yaml:"source"`
 	Format           string            `hcl:"format"`
@@ -77,8 +86,16 @@ func (c *NamespaceConfig) Compile() error {
 			return nil
 		}
 	}
+	if c.NamespaceLabelName != "" {
+		c.NamespaceLabels = make(map[string]string)
+		c.NamespaceLabels[c.NamespaceLabelName] = c.Name
+	}
 
 	c.OrderLabels()
+	c.NamespacePrefix = c.Name
+	if c.MetricsOverride != nil {
+		c.NamespacePrefix = c.MetricsOverride.Prefix
+	}
 
 	return nil
 }