From 99198c920ff0980c3b055588febb45c66e76bdc9 Mon Sep 17 00:00:00 2001 From: Eran Raichstein Date: Wed, 6 Apr 2022 15:34:01 +0300 Subject: [PATCH] add custom network service names --- README.md | 7 +- docs/api.md | 2 + go.mod | 2 +- pkg/api/transform_network.go | 2 + .../transform/network_services/netdb.go | 217 ++++++++++++++++++ pkg/pipeline/transform/transform_network.go | 12 +- .../transform/transform_network_test.go | 71 ++++++ 7 files changed, 309 insertions(+), 4 deletions(-) create mode 100644 pkg/pipeline/transform/network_services/netdb.go diff --git a/README.md b/README.md index 1bb190e17..b4f2bec01 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Default metrics are documented here [docs/metrics.md](docs/metrics.md). ```bash -Expose network flow-logs from metrics +Transform, persist and expose flow-logs as network metrics Usage: flowlogs-pipeline [flags] @@ -427,7 +427,10 @@ entries The third rule `add_service` generates a new field named `service` with the known network service name of `dstPort` port and `protocol` protocol. Unrecognized ports are ignored -> Note: `protocol` can be either network protocol name or number +> Note: `protocol` can be either network protocol name or number +> +> Note: optionally supports custom network services resolution by defining configuration parameters +> `servicesfile` and `protocolsfile` with paths to custom services/protocols files respectively The fourth rule `add_location` generates new fields with the geo-location information retrieved from DB [ip2location](https://lite.ip2location.com/) based on `dstIP` IP. diff --git a/docs/api.md b/docs/api.md index 1a82b4b01..7b6a177dd 100644 --- a/docs/api.md +++ b/docs/api.md @@ -116,6 +116,8 @@ Following is the supported API format for network transformations: add_kubernetes: add output kubernetes fields from input parameters: parameters specific to type kubeconfigpath: path to kubeconfig file (optional) + servicesfile: path to services file (optional, default: /etc/services) + protocolsfile: path to protocols file (optional, default: /etc/protocols) ## Write Loki API Following is the supported API format for writing to loki: diff --git a/go.mod b/go.mod index 409e235be..762bee48d 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( golang.org/x/net v0.0.0-20220225172249-27dd8689420f google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v2 v2.4.0 - honnef.co/go/netdb v0.0.0-20210921115105-e902e863d85d k8s.io/api v0.23.4 k8s.io/apimachinery v0.23.4 k8s.io/client-go v0.23.4 @@ -86,6 +85,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + honnef.co/go/netdb v0.0.0-20210921115105-e902e863d85d // indirect k8s.io/klog/v2 v2.30.0 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect diff --git a/pkg/api/transform_network.go b/pkg/api/transform_network.go index 631c2d1ec..305547f9a 100644 --- a/pkg/api/transform_network.go +++ b/pkg/api/transform_network.go @@ -20,6 +20,8 @@ package api type TransformNetwork struct { Rules NetworkTransformRules `yaml:"rules" doc:"list of transform rules, each includes:"` KubeConfigPath string `yaml:"kubeconfigpath" doc:"path to kubeconfig file (optional)"` + ServicesFile string `yaml:"servicesfile" doc:"path to services file (optional, default: /etc/services)"` + ProtocolsFile string `yaml:"protocolsfile" doc:"path to protocols file (optional, default: /etc/protocols)"` } type TransformNetworkOperationEnum struct { diff --git a/pkg/pipeline/transform/network_services/netdb.go b/pkg/pipeline/transform/network_services/netdb.go new file mode 100644 index 000000000..a0853886f --- /dev/null +++ b/pkg/pipeline/transform/network_services/netdb.go @@ -0,0 +1,217 @@ +/* + * Copyright (C) 2022 IBM, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * > Note: this code is a revised and enhanced version of the netdb.go file + * > from https://github.com/dominikh/go-netdb/ (MIT License) + */ + +package netdb + +import ( + "io/ioutil" + "strconv" + "strings" +) + +type Protoent struct { + Name string + Aliases []string + Number int +} + +type Servent struct { + Name string + Aliases []string + Port int + Protocol *Protoent +} + +// These variables get populated from /etc/protocols and /etc/services +// respectively. +var ( + Protocols []*Protoent + Services []*Servent +) + +func Init(protocolsFile, servicesFile string) error { + Protocols = []*Protoent{} + Services = []*Servent{} + + protoMap := make(map[string]*Protoent) + + // Load protocols + if protocolsFile == "" { + protocolsFile = "/etc/protocols" + } + data, err := ioutil.ReadFile(protocolsFile) + if err != nil { + return err + } + + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + split := strings.SplitN(line, "#", 2) + fields := strings.Fields(split[0]) + if len(fields) < 2 { + continue + } + + num, err := strconv.ParseInt(fields[1], 10, 32) + if err != nil { + return err + } + + protoent := &Protoent{ + Name: fields[0], + Aliases: fields[2:], + Number: int(num), + } + Protocols = append(Protocols, protoent) + + protoMap[fields[0]] = protoent + } + + // Load services + if servicesFile == "" { + servicesFile = "/etc/services" + } + data, err = ioutil.ReadFile(servicesFile) + if err != nil { + return err + } + + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + split := strings.SplitN(line, "#", 2) + fields := strings.Fields(split[0]) + if len(fields) < 2 { + continue + } + + name := fields[0] + portproto := strings.SplitN(fields[1], "/", 2) + port, err := strconv.ParseInt(portproto[0], 10, 32) + if err != nil { + return err + } + + proto := portproto[1] + aliases := fields[2:] + + Services = append(Services, &Servent{ + Name: name, + Aliases: aliases, + Port: int(port), + Protocol: protoMap[proto], + }) + } + + return nil +} + +// Equal checks if two Protoents are the same, which is the case if +// their protocol numbers are identical or when both Protoents are +// nil. +func (this *Protoent) Equal(other *Protoent) bool { + if this == nil && other == nil { + return true + } + + if this == nil || other == nil { + return false + } + + return this.Number == other.Number +} + +// Equal checks if two Servents are the same, which is the case if +// their port numbers and protocols are identical or when both +// Servents are nil. +func (this *Servent) Equal(other *Servent) bool { + if this == nil && other == nil { + return true + } + + if this == nil || other == nil { + return false + } + + return this.Port == other.Port && + this.Protocol.Equal(other.Protocol) +} + +// GetProtoByNumber returns the Protoent for a given protocol number. +func GetProtoByNumber(num int) (protoent *Protoent) { + for _, protoent := range Protocols { + if protoent.Number == num { + return protoent + } + } + return nil +} + +// GetProtoByName returns the Protoent whose name or any of its +// aliases matches the argument. +func GetProtoByName(name string) (protoent *Protoent) { + for _, protoent := range Protocols { + if protoent.Name == name { + return protoent + } + + for _, alias := range protoent.Aliases { + if alias == name { + return protoent + } + } + } + + return nil +} + +// GetServByName returns the Servent for a given service name or alias +// and protocol. If the protocol is nil, the first service matching +// the service name is returned. +func GetServByName(name string, protocol *Protoent) (servent *Servent) { + for _, servent := range Services { + if !servent.Protocol.Equal(protocol) { + continue + } + + if servent.Name == name { + return servent + } + + for _, alias := range servent.Aliases { + if alias == name { + return servent + } + } + } + + return nil +} + +// GetServByPort returns the Servent for a given port number and +// protocol. If the protocol is nil, the first service matching the +// port number is returned. +func GetServByPort(port int, protocol *Protoent) *Servent { + for _, servent := range Services { + if servent.Port == port && servent.Protocol.Equal(protocol) { + return servent + } + } + + return nil +} diff --git a/pkg/pipeline/transform/transform_network.go b/pkg/pipeline/transform/transform_network.go index 0052a683c..c550f826d 100644 --- a/pkg/pipeline/transform/transform_network.go +++ b/pkg/pipeline/transform/transform_network.go @@ -31,8 +31,8 @@ import ( "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/connection_tracking" "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/kubernetes" "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/location" + netdb "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/network_services" log "github.com/sirupsen/logrus" - "honnef.co/go/netdb" ) type Network struct { @@ -168,6 +168,7 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) { var needToInitLocationDB = false var needToInitKubeData = false var needToInitConnectionTracking = false + var needToInitNetworkServices = false jsonNetworkTransform := params.Transform.Network for _, rule := range jsonNetworkTransform.Rules { @@ -178,6 +179,8 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) { needToInitKubeData = true case api.TransformNetworkOperationName("ConnTracking"): needToInitConnectionTracking = true + case api.TransformNetworkOperationName("AddService"): + needToInitNetworkServices = true } } @@ -199,6 +202,13 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) { } } + if needToInitNetworkServices { + err := netdb.Init(jsonNetworkTransform.ProtocolsFile, jsonNetworkTransform.ServicesFile) + if err != nil { + return nil, err + } + } + return &Network{ api.TransformNetwork{ Rules: jsonNetworkTransform.Rules, diff --git a/pkg/pipeline/transform/transform_network_test.go b/pkg/pipeline/transform/transform_network_test.go index 14b0a6a75..9144591a0 100644 --- a/pkg/pipeline/transform/transform_network_test.go +++ b/pkg/pipeline/transform/transform_network_test.go @@ -18,11 +18,13 @@ package transform import ( + "os" "testing" "github.com/netobserv/flowlogs-pipeline/pkg/api" "github.com/netobserv/flowlogs-pipeline/pkg/config" "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/location" + netdb "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/network_services" "github.com/netobserv/flowlogs-pipeline/pkg/test" "github.com/stretchr/testify/require" ) @@ -134,6 +136,9 @@ func Test_Transform(t *testing.T) { err := location.InitLocationDB() require.NoError(t, err) + err = netdb.Init("", "") + require.NoError(t, err) + output := networkTransform.Transform([]config.GenericMap{entry}) require.Equal(t, expectedOutput, output[0]) @@ -157,6 +162,9 @@ func Test_TransformAddSubnetParseCIDRFailure(t *testing.T) { err := location.InitLocationDB() require.NoError(t, err) + err = netdb.Init("", "") + require.NoError(t, err) + output := networkTransform.Transform([]config.GenericMap{entry}) require.Equal(t, expectedOutput, output[0]) @@ -313,3 +321,66 @@ func Test_Transform_AddIfScientificNotation(t *testing.T) { require.Equal(t, true, output[0]["smaller_than_10_Evaluate"]) require.Equal(t, 1.2345e-67, output[0]["smaller_than_10"]) } + +func Test_TransformNetworkOperationNameCustomServices(t *testing.T) { + rules := api.NetworkTransformRules{ + api.NetworkTransformRule{ + Input: "dstPort", + Output: "service", + Type: "add_service", + Parameters: "protocol", + }, + } + + entry := test.GetIngestMockEntry(false) + + var networkTransform = Network{ + api.TransformNetwork{ + Rules: rules, + }, + } + + err := netdb.Init("", "") + require.NoError(t, err) + output := networkTransform.Transform([]config.GenericMap{entry}) + require.Equal(t, "ssh", output[0]["service"]) + + // check custom services file + customService := "custom_ssh 22/tcp # The Secure Shell (SSH) Protocol\n" + err = os.WriteFile("/tmp/services", []byte(customService), 0644) + require.NoError(t, err) + + err = netdb.Init("", "/tmp/services") + require.NoError(t, err) + output = networkTransform.Transform([]config.GenericMap{entry}) + require.Equal(t, "custom_ssh", output[0]["service"]) + + rules = api.NetworkTransformRules{ + api.NetworkTransformRule{ + Input: "dstPort", + Output: "service_protocol_num", + Type: "add_service", + Parameters: "protocol_num", + }, + } + + networkTransform = Network{ + api.TransformNetwork{ + Rules: rules, + }, + } + + // check custom protocol file + customProtocol := "custom_tcp 6 CUSTOM_TCP # transmission control protocol\n" + err = os.WriteFile("/tmp/protocols", []byte(customProtocol), 0644) + require.NoError(t, err) + + customService = "custom_ssh 22/custom_TCP # The Secure Shell (SSH) Protocol\n" + err = os.WriteFile("/tmp/services", []byte(customService), 0644) + require.NoError(t, err) + + err = netdb.Init("/tmp/protocols", "/tmp/services") + require.NoError(t, err) + output = networkTransform.Transform([]config.GenericMap{entry}) + require.Equal(t, "custom_ssh", output[0]["service_protocol_num"]) +}