diff --git a/cmd/confgenerator/main.go b/cmd/confgenerator/main.go index d32e00535..110eb1916 100644 --- a/cmd/confgenerator/main.go +++ b/cmd/confgenerator/main.go @@ -63,7 +63,7 @@ func initConfig() { if err != nil { log.Fatal(err) } - // Search config in home directory with name ".flpconfgen" (without extension). + // Search config in home directory with name ".confgen" (without extension). v.AddConfigPath(home) v.SetConfigName(defaultLogFileName) } @@ -132,7 +132,8 @@ func initFlags() { rootCmd.PersistentFlags().StringVar(&confgen.Opt.DestConfFile, "destConfFile", "/tmp/flowlogs-pipeline.conf.yaml", "destination configuration file") rootCmd.PersistentFlags().StringVar(&confgen.Opt.DestDocFile, "destDocFile", "/tmp/metrics.md", "destination documentation file (.md)") rootCmd.PersistentFlags().StringVar(&confgen.Opt.DestGrafanaJsonnetFolder, "destGrafanaJsonnetFolder", "/tmp/jsonnet", "destination grafana jsonnet folder") - rootCmd.PersistentFlags().StringSliceVar(&confgen.Opt.SkipWithLabels, "skipWithLabels", nil, "Skip definitions with Labels") + rootCmd.PersistentFlags().StringSliceVar(&confgen.Opt.SkipWithTags, "skipWithTags", nil, "Skip definitions with Tags") + rootCmd.PersistentFlags().StringSliceVar(&confgen.Opt.GenerateStages, "generateStages", nil, "Produce only specified stages (ingest, transform_generic, transform_network, extract_aggregate, encode_prom, write_loki") } func main() { diff --git a/docs/confGenerator.md b/docs/confGenerator.md index e494bde64..30ce18709 100644 --- a/docs/confGenerator.md +++ b/docs/confGenerator.md @@ -21,13 +21,14 @@ Usage: Flags: --config string config file (default is $HOME/.confgen) - --destConfFile string destination configuration file (default "flowlogs-pipeline.conf.yaml") - --destGrafanaJsonnetFolder string destination grafana jsonnet folder + --destConfFile string destination configuration file (default "/tmp/flowlogs-pipeline.conf.yaml") + --destDocFile string destination documentation file (.md) (default "/tmp/metrics.md") + --destGrafanaJsonnetFolder string destination grafana jsonnet folder (default "/tmp/jsonnet") + --generateStages strings Produce only specified stages (ingest, transform_generic, transform_network, extract_aggregate, encode_prom, write_loki -h, --help help for confgenerator --log-level string Log level: debug, info, warning, error (default "error") - --skipWithLabels strings Skip definitions with Labels + --skipWithTags strings Skip definitions with Tags --srcFolder string source folder (default "network_definitions") - ``` > Note: confgenerator is available also from `netobserv/flowlogs-pipeline` quay image. To use execute: diff --git a/network_definitions/bandwidth_per_network_service.yaml b/network_definitions/bandwidth_per_network_service.yaml index 8c79bcb03..4373a7153 100644 --- a/network_definitions/bandwidth_per_network_service.yaml +++ b/network_definitions/bandwidth_per_network_service.yaml @@ -5,7 +5,7 @@ details: Sum bytes for all traffic per network service usage: Evaluate network usage breakdown per network service -labels: +tags: - bandwidth - graph - rate diff --git a/network_definitions/bandwidth_per_src_dest_subnet.yaml b/network_definitions/bandwidth_per_src_dest_subnet.yaml index 8e2efb897..44ef5a23a 100644 --- a/network_definitions/bandwidth_per_src_dest_subnet.yaml +++ b/network_definitions/bandwidth_per_src_dest_subnet.yaml @@ -5,7 +5,7 @@ details: Sum bandwidth bytes for all traffic per source / destination subnet pair usage: Evaluate network usage breakdown per source / destination subnet pair -labels: +tags: - bandwidth - graph - rate diff --git a/network_definitions/bandwidth_per_src_subnet.yaml b/network_definitions/bandwidth_per_src_subnet.yaml index ca376a3d7..d39832580 100644 --- a/network_definitions/bandwidth_per_src_subnet.yaml +++ b/network_definitions/bandwidth_per_src_subnet.yaml @@ -5,7 +5,7 @@ details: Sum bytes for all traffic per source subnet usage: Evaluate network usage breakdown per source subnet -labels: +tags: - bandwidth - graph - rate diff --git a/network_definitions/connection_rate_per_dest_subnet.yaml b/network_definitions/connection_rate_per_dest_subnet.yaml index f4a493bd6..06042a1d0 100644 --- a/network_definitions/connection_rate_per_dest_subnet.yaml +++ b/network_definitions/connection_rate_per_dest_subnet.yaml @@ -5,7 +5,7 @@ details: Counts the number of connections per subnet with network prefix length /16 (using conn_tracking sum isNewFlow field) usage: Evaluate network connections per subnet -labels: +tags: - rate - subnet transform: diff --git a/network_definitions/connection_rate_per_src_subnet.yaml b/network_definitions/connection_rate_per_src_subnet.yaml index b5a567ec7..ea7eca8fc 100644 --- a/network_definitions/connection_rate_per_src_subnet.yaml +++ b/network_definitions/connection_rate_per_src_subnet.yaml @@ -5,7 +5,7 @@ details: Counts the number of connections per subnet with network prefix length /16 usage: Evaluate network connections per subnet -labels: +tags: - rate - subnet transform: diff --git a/network_definitions/connection_rate_per_tcp_flags.yaml b/network_definitions/connection_rate_per_tcp_flags.yaml index 2f78b4a6d..7426b3654 100644 --- a/network_definitions/connection_rate_per_tcp_flags.yaml +++ b/network_definitions/connection_rate_per_tcp_flags.yaml @@ -5,7 +5,7 @@ details: Counts the number of connections per tcp flags usage: Evaluate difference in connections rate of different TCP Flags. Can be used, for example, to identify syn-attacks. -labels: +tags: - rate - TCPFlags extract: diff --git a/network_definitions/connections_per_dst_as.yaml b/network_definitions/connections_per_dst_as.yaml index 840a5cd4f..36b3376e1 100644 --- a/network_definitions/connections_per_dst_as.yaml +++ b/network_definitions/connections_per_dst_as.yaml @@ -5,7 +5,7 @@ details: Aggregates flow records by values of "DstAS" field and counts the number of entries in each aggregate with non zero value usage: Evaluate amount of connections targeted at different Autonomous Systems -labels: +tags: - rate - count - AS diff --git a/network_definitions/connections_per_src_as.yaml b/network_definitions/connections_per_src_as.yaml index ab6122694..b883e780e 100644 --- a/network_definitions/connections_per_src_as.yaml +++ b/network_definitions/connections_per_src_as.yaml @@ -5,7 +5,7 @@ details: Aggregates flow records by values of "SrcAS" field and counts the number of entries in each aggregate with non zero value usage: Evaluate amount of connections initiated by different Autonomous Systems -labels: +tags: - rate - count - AS diff --git a/network_definitions/count_per_src_dest_subnet.yaml b/network_definitions/count_per_src_dest_subnet.yaml index 5b92651a4..80f4be3cc 100644 --- a/network_definitions/count_per_src_dest_subnet.yaml +++ b/network_definitions/count_per_src_dest_subnet.yaml @@ -5,7 +5,7 @@ details: Count the number of distinct source / destination subnet pairs usage: Evaluate network usage breakdown per source / destination subnet pair -labels: +tags: - count - graph - rate diff --git a/network_definitions/egress_bandwidth_per_dest_subnet.yaml b/network_definitions/egress_bandwidth_per_dest_subnet.yaml index 9d2f8009a..6e83e0c73 100644 --- a/network_definitions/egress_bandwidth_per_dest_subnet.yaml +++ b/network_definitions/egress_bandwidth_per_dest_subnet.yaml @@ -5,7 +5,7 @@ details: Sum egress bytes for all traffic per destination subnet usage: Evaluate network usage breakdown per destination subnet -labels: +tags: - bandwidth - graph - rate diff --git a/network_definitions/egress_bandwidth_per_namespace.yaml b/network_definitions/egress_bandwidth_per_namespace.yaml index 2b87ffada..6c6e18d72 100644 --- a/network_definitions/egress_bandwidth_per_namespace.yaml +++ b/network_definitions/egress_bandwidth_per_namespace.yaml @@ -5,7 +5,7 @@ details: Sum egress bytes for all traffic per namespace usage: Evaluate network usage breakdown per namespace -labels: +tags: - kubernetes - bandwidth - graph diff --git a/network_definitions/flows_length_histogram.yaml b/network_definitions/flows_length_histogram.yaml index 7dc7deff4..e472304ef 100644 --- a/network_definitions/flows_length_histogram.yaml +++ b/network_definitions/flows_length_histogram.yaml @@ -5,7 +5,7 @@ details: Flows length distribution over time usage: Evaluate flows length behavior including mice/elephant use-case -labels: +tags: - bandwidth - mice - elephant diff --git a/network_definitions/geo-location_rate_per_dest.yaml b/network_definitions/geo-location_rate_per_dest.yaml index 2439772de..cea5ca2b4 100644 --- a/network_definitions/geo-location_rate_per_dest.yaml +++ b/network_definitions/geo-location_rate_per_dest.yaml @@ -5,7 +5,7 @@ details: Counts the number of connections per geo-location based on destination IP usage: Evaluate network connections geo-location -labels: +tags: - rate - connections-count - geo-location diff --git a/network_definitions/loki_bandwidth_per_namespace.yaml b/network_definitions/loki_bandwidth_per_namespace.yaml index 9ae4c9ada..414b3b9bf 100644 --- a/network_definitions/loki_bandwidth_per_namespace.yaml +++ b/network_definitions/loki_bandwidth_per_namespace.yaml @@ -5,7 +5,7 @@ details: Sum bytes for all traffic per source namespace usage: Evaluate network usage breakdown per source namespace -labels: +tags: - loki - graph - rate diff --git a/network_definitions/loki_logs_per_sec.yaml b/network_definitions/loki_logs_per_sec.yaml index a06dba6f6..18d01d987 100644 --- a/network_definitions/loki_logs_per_sec.yaml +++ b/network_definitions/loki_logs_per_sec.yaml @@ -5,7 +5,7 @@ details: Rate of loki logs per sec usage: Evaluate loki service usage -labels: +tags: - loki - graph - rate diff --git a/network_definitions/network_services_count.yaml b/network_definitions/network_services_count.yaml index bb39dce0d..00ea95650 100644 --- a/network_definitions/network_services_count.yaml +++ b/network_definitions/network_services_count.yaml @@ -5,7 +5,7 @@ details: Counts the number of connections per network service based on destination port number and protocol usage: Evaluate network services -labels: +tags: - rate - network-services - destination-port diff --git a/pkg/confgen/confgen.go b/pkg/confgen/confgen.go index bebc6389a..aac44ca94 100644 --- a/pkg/confgen/confgen.go +++ b/pkg/confgen/confgen.go @@ -40,7 +40,7 @@ type Definition struct { Description string Details string Usage string - Labels []string + Tags []string TransformNetwork *api.TransformNetwork AggregateDefinitions *aggregate.Definitions PromEncode *api.PromEncode @@ -62,7 +62,7 @@ type DefFile struct { Description string `yaml:"description"` Details string `yaml:"details"` Usage string `yaml:"usage"` - Labels []string `yaml:"labels"` + Tags []string `yaml:"tags"` Transform map[string]interface{} `yaml:"transform"` Extract map[string]interface{} `yaml:"extract"` Encode map[string]interface{} `yaml:"encode"` @@ -88,10 +88,21 @@ func (cg *ConfGen) Run() error { cg.dedupe() - err = cg.generateFlowlogs2PipelineConfig(Opt.DestConfFile) - if err != nil { - log.Debugf("cg.generateFlowlogs2PipelineConfig err: %v ", err) - return err + if len(Opt.GenerateStages) != 0 { + config := cg.GenerateTruncatedConfig(Opt.GenerateStages) + err = cg.writeConfigFile(Opt.DestConfFile, config) + if err != nil { + log.Debugf("cg.GenerateTruncatedConfig err: %v ", err) + return err + } + return nil + } else { + config := cg.GenerateFlowlogs2PipelineConfig() + err = cg.writeConfigFile(Opt.DestConfFile, config) + if err != nil { + log.Debugf("cg.GenerateFlowlogs2PipelineConfig err: %v ", err) + return err + } } err = cg.generateDoc(Opt.DestDocFile) @@ -153,11 +164,11 @@ func (cg *ConfGen) parseFile(fileName string) error { return err } - //skip if there skip label match - for _, skipLabel := range Opt.SkipWithLabels { - for _, label := range defFile.Labels { - if skipLabel == label { - return fmt.Errorf("skipping definition %s due to skip label %s", fileName, label) + //skip if their skip tag match + for _, skipTag := range Opt.SkipWithTags { + for _, tag := range defFile.Tags { + if skipTag == tag { + return fmt.Errorf("skipping definition %s due to skip tag %s", fileName, tag) } } } @@ -168,7 +179,7 @@ func (cg *ConfGen) parseFile(fileName string) error { Description: defFile.Description, Details: defFile.Details, Usage: defFile.Usage, - Labels: defFile.Labels, + Tags: defFile.Tags, } // parse transport diff --git a/pkg/confgen/config.go b/pkg/confgen/config.go index 29e5434fa..31e0ce1b1 100644 --- a/pkg/confgen/config.go +++ b/pkg/confgen/config.go @@ -30,7 +30,8 @@ type Options struct { DestDocFile string DestGrafanaJsonnetFolder string SrcFolder string - SkipWithLabels []string + SkipWithTags []string + GenerateStages []string } var ( diff --git a/pkg/confgen/doc.go b/pkg/confgen/doc.go index d9d9d1025..a3b03fd51 100644 --- a/pkg/confgen/doc.go +++ b/pkg/confgen/doc.go @@ -70,7 +70,7 @@ func (cg *ConfGen) generateDoc(fileName string) error { replacer := strings.NewReplacer("-", " ", "_", " ") name := replacer.Replace(filepath.Base(metric.FileName[:len(metric.FileName)-len(filepath.Ext(metric.FileName))])) - labels := strings.Join(metric.Labels[:], ", ") + labels := strings.Join(metric.Tags[:], ", ") // TODO: add support for multiple operations operation := cg.generateOperationText(*metric.AggregateDefinitions) expose := cg.generatePromEncodeText(metric.PromEncode.Metrics) @@ -82,7 +82,7 @@ func (cg *ConfGen) generateDoc(fileName string) error { |:---|:---| | **Details** | %s | | **Usage** | %s | -| **Labels** | %s | +| **Tags** | %s | %s%s%s||| `, diff --git a/pkg/confgen/flowlogs2metrics_config.go b/pkg/confgen/flowlogs2metrics_config.go index d2ca415c4..ca9d03981 100644 --- a/pkg/confgen/flowlogs2metrics_config.go +++ b/pkg/confgen/flowlogs2metrics_config.go @@ -21,10 +21,11 @@ import ( "fmt" "io/ioutil" + log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) -func (cg *ConfGen) generateFlowlogs2PipelineConfig(fileName string) error { +func (cg *ConfGen) GenerateFlowlogs2PipelineConfig() map[string]interface{} { config := map[string]interface{}{ "log-level": "error", "pipeline": []map[string]string{ @@ -97,7 +98,84 @@ func (cg *ConfGen) generateFlowlogs2PipelineConfig(fileName string) error { }, }, } + return config +} + +func (cg *ConfGen) GenerateTruncatedConfig(stages []string) map[string]interface{} { + parameters := make([]map[string]interface{}, len(stages)) + for i, stage := range stages { + switch stage { + case "ingest": + parameters[i] = map[string]interface{}{ + "name": "ingest_collector", + "ingest": map[string]interface{}{ + "type": "collector", + "collector": map[string]interface{}{ + "port": cg.config.Ingest.Collector.Port, + "portLegacy": cg.config.Ingest.Collector.PortLegacy, + "hostname": cg.config.Ingest.Collector.HostName, + }, + }, + } + case "transform_generic": + parameters[i] = map[string]interface{}{ + "name": "transform_generic", + "transform": map[string]interface{}{ + "type": "generic", + "generic": map[string]interface{}{ + "policy": "replace_keys", + "rules": cg.config.Transform.Generic.Rules, + }, + }, + } + case "transform_network": + parameters[i] = map[string]interface{}{ + "name": "transform_network", + "transform": map[string]interface{}{ + "type": "network", + "network": map[string]interface{}{ + "rules": cg.transformRules, + }, + }, + } + case "extract_aggregate": + parameters[i] = map[string]interface{}{ + "name": "extract_aggregate", + "extract": map[string]interface{}{ + "type": "aggregates", + "aggregates": cg.aggregateDefinitions, + }, + } + case "encode_prom": + parameters[i] = map[string]interface{}{ + "name": "encode_prom", + "encode": map[string]interface{}{ + "type": "prom", + "prom": map[string]interface{}{ + "port": cg.config.Encode.Prom.Port, + "prefix": cg.config.Encode.Prom.Prefix, + "metrics": cg.promMetrics, + }, + }, + } + case "write_loki": + parameters[i] = map[string]interface{}{ + "name": "write_loki", + "write": map[string]interface{}{ + "type": cg.config.Write.Type, + "loki": cg.config.Write.Loki, + }, + } + } + } + log.Debugf("parameters = %v \n", parameters) + config := map[string]interface{}{ + "parameters": parameters, + } + return config +} +func (cg *ConfGen) writeConfigFile(fileName string, config map[string]interface{}) error { configData, err := yaml.Marshal(&config) if err != nil { return err