Add filter field to prometheus encoder

contrib/kubernetes/flowlogs-pipeline.conf.yaml

@@ -165,105 +165,150 @@ parameters:
       metrics:
       - name: bandwidth_per_network_service
         type: counter
-        valuekey: bandwidth_network_service_recent_op_value
+        filter:
+          key: name
+          value: bandwidth_network_service
+        valuekey: recent_op_value
         labels:
         - by
         - aggregate
         buckets: []
       - name: bandwidth_per_source_destination_subnet
         type: counter
-        valuekey: bandwidth_source_destination_subnet_recent_op_value
+        filter:
+          key: name
+          value: bandwidth_source_destination_subnet
+        valuekey: recent_op_value
         labels:
         - by
         - aggregate
         buckets: []
       - name: bandwidth_per_source_subnet
         type: counter
-        valuekey: bandwidth_source_subnet_recent_op_value
+        filter:
+          key: name
+          value: bandwidth_source_subnet
+        valuekey: recent_op_value
         labels:
         - by
         - aggregate
         buckets: []
       - name: connections_per_destination_subnet
         type: counter
-        valuekey: dest_connection_subnet_recent_count
+        filter:
+          key: name
+          value: dest_connection_subnet_count
+        valuekey: recent_count
         labels:
         - by
         - aggregate
         buckets: []
       - name: connections_per_source_subnet
         type: counter
-        valuekey: src_connection_count_recent_count
+        filter:
+          key: name
+          value: src_connection_count
+        valuekey: recent_count
         labels:
         - by
         - aggregate
         buckets: []
       - name: connections_per_tcp_flags
         type: counter
-        valuekey: TCPFlags_recent_count
+        filter:
+          key: name
+          value: TCPFlags_count
+        valuekey: recent_count
         labels:
         - by
         - aggregate
         buckets: []
       - name: connections_per_destination_as
         type: counter
-        valuekey: dst_as_connection_recent_count
+        filter:
+          key: name
+          value: dst_as_connection_count
+        valuekey: recent_count
         labels:
         - by
         - aggregate
         buckets: []
       - name: connections_per_source_as
         type: counter
-        valuekey: src_as_connection_recent_count
+        filter:
+          key: name
+          value: src_as_connection_count
+        valuekey: recent_count
         labels:
         - by
         - aggregate
         buckets: []
       - name: count_per_source_destination_subnet
         type: counter
-        valuekey: count_source_destination_subnet_recent_count
+        filter:
+          key: name
+          value: count_source_destination_subnet
+        valuekey: recent_count
         labels:
         - by
         - aggregate
         buckets: []
       - name: egress_per_destination_subnet
         type: counter
-        valuekey: bandwidth_destination_subnet_recent_op_value
+        filter:
+          key: name
+          value: bandwidth_destination_subnet
+        valuekey: recent_op_value
         labels:
         - by
         - aggregate
         buckets: []
       - name: egress_per_namespace
         type: counter
-        valuekey: bandwidth_namespace_recent_op_value
+        filter:
+          key: name
+          value: bandwidth_namespace
+        valuekey: recent_op_value
         labels:
         - by
         - aggregate
         buckets: []
       - name: connections_per_destination_location
         type: counter
-        valuekey: dest_connection_location_recent_count
+        filter:
+          key: name
+          value: dest_connection_location_count
+        valuekey: recent_count
         labels:
         - by
         - aggregate
         buckets: []
       - name: mice_count
         type: counter
-        valuekey: mice_count_recent_count
+        filter:
+          key: name
+          value: mice_count
+        valuekey: recent_count
         labels:
         - by
         - aggregate
         buckets: []
       - name: elephant_count
         type: counter
-        valuekey: elephant_count_recent_count
+        filter:
+          key: name
+          value: elephant_count
+        valuekey: recent_count
         labels:
         - by
         - aggregate
         buckets: []
       - name: service_count
         type: counter
-        valuekey: dest_service_recent_count
+        filter:
+          key: name
+          value: dest_service_count
+        valuekey: recent_count
         labels:
         - by
         - aggregate

docs/api.md

@@ -10,6 +10,9 @@ Following is the supported API format for prometheus encode:
                      gauge: single numerical value that can arbitrarily go up and down
                      counter: monotonically increasing counter whose value can only increase
                      histogram: counts samples in configurable buckets
+                 filter: the criterion to filter entries by
+                     key: the key to match and filter by
+                     value: the value to match and filter by
                  valuekey: entry key from which to resolve metric value
                  labels: labels to be associated with the metric
                  buckets: histogram buckets

docs/confGenerator.md

@@ -63,11 +63,11 @@ make local-redeploy
 ```
 > Note: Additional information on usage and deployment can be found in flowlogs-pipeline README  
 
-> Note: learning from examples and existing metric definitions is very-useful
+> Note: Learning from examples and existing metric definitions is very useful.
 
 ### Network definition explained
 
-In this section we explain how network definition are structured.  This is useful for development of 
+In this section we explain how network definition is structured. This is useful for development of 
 new network definitions as well as debugging and working with existing network definition.
 
 ```shell
@@ -101,8 +101,9 @@ encode: (9)
     metrics:
       - name: metricName (9.2)
         type: metricType (9.3)
-        valuekey: aggregate_name_value (9.4)
-        labels: (9.5)
+        filter: {key: myKey, value: myValue} (9.4)
+        valuekey: value (9.5)
+        labels: (9.6)
           - by
           - aggregate
 visualization: (10)
@@ -133,12 +134,15 @@ this actually moves the data from being log lines into being a metric named (8.2
 > For additional details on `extract aggregates`
 > refer to [README.md](../README.md#aggregates).  
 
-(9) Next, the metrics from (8.2) are sent to prometheus (9.1). Make sure that (9.4) value is
-set to the metric name from (8.2) with suffix `_value`. 
+(9) Next, the metrics from (8.2) are sent to prometheus (9.1). 
 The metric name in prometheus will be called as the value of (9.2) with 
 the prefix from the `config.yaml` file. 
-The type of the prometheus metric will be (9.3) (e.g. gauge). 
-Prometheus will add labels to the metric based on the (9.5) fields.  
+The type of the prometheus metric will be (9.3) (e.g. gauge, counter or histogram). 
+The filter field (9.4) determines which aggregates will take into account. 
+The key should be `"name"` and the value should match the aggregate name (8.2)
+The value to be used by prometheus is taken from the field defined in (9.5).
+For `Gauges`, use `total_value` or `total_count`. For `Counters`, use `recent_op_value` or `recent_count`.
+Prometheus will add labels to the metric based on the (9.6) fields. 
 
 (10) next, using grafana to visualize the metric with name from (9.2) including the 
 prefix and using the prometheus expression from (10.1). 

network_definitions/bandwidth_per_network_service.yaml

@@ -29,7 +29,8 @@ encode:
     metrics:
       - name: bandwidth_per_network_service
         type: counter
-        valuekey: bandwidth_network_service_recent_op_value
+        filter: {key: name, value: bandwidth_network_service}
+        valuekey: recent_op_value
         labels:
           - by
           - aggregate

network_definitions/bandwidth_per_src_dest_subnet.yaml

@@ -34,7 +34,8 @@ encode:
     metrics:
       - name: bandwidth_per_source_destination_subnet
         type: counter
-        valuekey: bandwidth_source_destination_subnet_recent_op_value
+        filter: {key: name, value: bandwidth_source_destination_subnet}
+        valuekey: recent_op_value
         labels:
           - by
           - aggregate

network_definitions/bandwidth_per_src_subnet.yaml

@@ -29,7 +29,8 @@ encode:
     metrics:
       - name: bandwidth_per_source_subnet
         type: counter
-        valuekey: bandwidth_source_subnet_recent_op_value
+        filter: {key: name, value: bandwidth_source_subnet}
+        valuekey: recent_op_value
         labels:
           - by
           - aggregate

network_definitions/connection_rate_per_dest_subnet.yaml

@@ -32,7 +32,8 @@ encode:
     metrics:
       - name: connections_per_destination_subnet
         type: counter
-        valuekey: dest_connection_subnet_recent_count
+        filter: {key: name, value: dest_connection_subnet_count}
+        valuekey: recent_count
         labels:
           - by
           - aggregate

network_definitions/connection_rate_per_src_subnet.yaml

@@ -27,7 +27,8 @@ encode:
     metrics:
       - name: connections_per_source_subnet
         type: counter
-        valuekey: src_connection_count_recent_count
+        filter: {key: name, value: src_connection_count}
+        valuekey: recent_count
         labels:
           - by
           - aggregate

network_definitions/connection_rate_per_tcp_flags.yaml

@@ -21,7 +21,8 @@ encode:
     metrics:
       - name: connections_per_tcp_flags
         type: counter
-        valuekey: TCPFlags_recent_count
+        filter: { key: name, value: TCPFlags_count }
+        valuekey: recent_count
         labels:
           - by
           - aggregate

network_definitions/connections_per_dst_as.yaml

@@ -22,7 +22,8 @@ encode:
     metrics:
       - name: connections_per_destination_as
         type: counter
-        valuekey: dst_as_connection_recent_count
+        filter: { key: name, value: dst_as_connection_count }
+        valuekey: recent_count
         labels:
           - by
           - aggregate

network_definitions/connections_per_src_as.yaml

@@ -22,7 +22,8 @@ encode:
     metrics:
       - name: connections_per_source_as
         type: counter
-        valuekey: src_as_connection_recent_count
+        filter: { key: name, value: src_as_connection_count }
+        valuekey: recent_count
         labels:
           - by
           - aggregate

network_definitions/count_per_src_dest_subnet.yaml

@@ -33,7 +33,8 @@ encode:
     metrics:
       - name: count_per_source_destination_subnet
         type: counter
-        valuekey: count_source_destination_subnet_recent_count
+        filter: { key: name, value: count_source_destination_subnet }
+        valuekey: recent_count
         labels:
           - by
           - aggregate

network_definitions/egress_bandwidth_per_dest_subnet.yaml

@@ -29,7 +29,8 @@ encode:
     metrics:
       - name: egress_per_destination_subnet
         type: counter
-        valuekey: bandwidth_destination_subnet_recent_op_value
+        filter: { key: name, value: bandwidth_destination_subnet }
+        valuekey: recent_op_value
         labels:
           - by
           - aggregate

network_definitions/egress_bandwidth_per_namespace.yaml

@@ -29,7 +29,8 @@ encode:
     metrics:
       - name: egress_per_namespace
         type: counter
-        valuekey: bandwidth_namespace_recent_op_value
+        filter: { key: name, value: bandwidth_namespace }
+        valuekey: recent_op_value
         labels:
           - by
           - aggregate

network_definitions/geo-location_rate_per_dest.yaml

@@ -28,7 +28,8 @@ encode:
     metrics:
       - name: connections_per_destination_location
         type: counter
-        valuekey: dest_connection_location_recent_count
+        filter: { key: name, value: dest_connection_location_count }
+        valuekey: recent_count
         labels:
           - by
           - aggregate

network_definitions/mice_elephants.yaml

@@ -36,13 +36,15 @@ encode:
     metrics:
       - name: mice_count
         type: counter
-        valuekey: mice_count_recent_count
+        filter: { key: name, value: mice_count }
+        valuekey: recent_count
         labels:
           - by
           - aggregate
       - name: elephant_count
         type: counter
-        valuekey: elephant_count_recent_count
+        filter: { key: name, value: elephant_count }
+        valuekey: recent_count
         labels:
           - by
           - aggregate

network_definitions/network_services_count.yaml

@@ -29,7 +29,8 @@ encode:
     metrics:
       - name: service_count
         type: counter
-        valuekey: dest_service_recent_count
+        filter: { key: name, value: dest_service_count }
+        valuekey: recent_count
         labels:
           - by
           - aggregate

pkg/api/encode_prom.go

@@ -35,11 +35,17 @@ func PromEncodeOperationName(operation string) string {
 }
 
 type PromMetricsItem struct {
-	Name     string    `yaml:"name" doc:"the metric name"`
-	Type     string    `yaml:"type" enum:"PromEncodeOperationEnum" doc:"one of the following:"`
-	ValueKey string    `yaml:"valuekey" doc:"entry key from which to resolve metric value"`
-	Labels   []string  `yaml:"labels" doc:"labels to be associated with the metric"`
-	Buckets  []float64 `yaml:"buckets" doc:"histogram buckets"`
+	Name     string            `yaml:"name" doc:"the metric name"`
+	Type     string            `yaml:"type" enum:"PromEncodeOperationEnum" doc:"one of the following:"`
+	Filter   PromMetricsFilter `yaml:"filter" doc:"the criterion to filter entries by"`
+	ValueKey string            `yaml:"valuekey" doc:"entry key from which to resolve metric value"`
+	Labels   []string          `yaml:"labels" doc:"labels to be associated with the metric"`
+	Buckets  []float64         `yaml:"buckets" doc:"histogram buckets"`
 }
 
 type PromMetricsItems []PromMetricsItem
+
+type PromMetricsFilter struct {
+	Key   string `yaml:"key" doc:"the key to match and filter by"`
+	Value string `yaml:"value" doc:"the value to match and filter by"`
+}