Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ Default metrics are documented here [docs/metrics.md](docs/metrics.md).

<!---AUTO-flowlogs-pipeline_help--->
```bash
Expose network flow-logs from metrics
Transform, persist and expose flow-logs as network metrics

Usage:
flowlogs-pipeline [flags]
Expand Down Expand Up @@ -427,7 +427,10 @@ entries

The third rule `add_service` generates a new field named `service` with the known network
service name of `dstPort` port and `protocol` protocol. Unrecognized ports are ignored
> Note: `protocol` can be either network protocol name or number
> Note: `protocol` can be either network protocol name or number
>
> Note: optionally supports custom network services resolution by defining configuration parameters
> `servicesfile` and `protocolsfile` with paths to custom services/protocols files respectively

The fourth rule `add_location` generates new fields with the geo-location information retrieved
from DB [ip2location](https://lite.ip2location.com/) based on `dstIP` IP.
Expand Down
2 changes: 2 additions & 0 deletions docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ Following is the supported API format for network transformations:
add_kubernetes: add output kubernetes fields from input
parameters: parameters specific to type
kubeconfigpath: path to kubeconfig file (optional)
servicesfile: path to services file (optional, default: /etc/services)
protocolsfile: path to protocols file (optional, default: /etc/protocols)
</pre>
## Write Loki API
Following is the supported API format for writing to loki:
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ require (
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
google.golang.org/protobuf v1.27.1
gopkg.in/yaml.v2 v2.4.0
honnef.co/go/netdb v0.0.0-20210921115105-e902e863d85d
k8s.io/api v0.23.4
k8s.io/apimachinery v0.23.4
k8s.io/client-go v0.23.4
Expand Down Expand Up @@ -86,6 +85,7 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.66.2 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
honnef.co/go/netdb v0.0.0-20210921115105-e902e863d85d // indirect
k8s.io/klog/v2 v2.30.0 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
Expand Down
2 changes: 2 additions & 0 deletions pkg/api/transform_network.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ package api
type TransformNetwork struct {
Rules NetworkTransformRules `yaml:"rules" doc:"list of transform rules, each includes:"`
KubeConfigPath string `yaml:"kubeconfigpath" doc:"path to kubeconfig file (optional)"`
ServicesFile string `yaml:"servicesfile" doc:"path to services file (optional, default: /etc/services)"`
ProtocolsFile string `yaml:"protocolsfile" doc:"path to protocols file (optional, default: /etc/protocols)"`
}

type TransformNetworkOperationEnum struct {
Expand Down
217 changes: 217 additions & 0 deletions pkg/pipeline/transform/network_services/netdb.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
/*
* Copyright (C) 2022 IBM, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* > Note: this code is a revised and enhanced version of the netdb.go file
* > from https://github.com/dominikh/go-netdb/ (MIT License)
*/

package netdb

import (
"io/ioutil"
"strconv"
"strings"
)

type Protoent struct {
Name string
Aliases []string
Number int
}

type Servent struct {
Name string
Aliases []string
Port int
Protocol *Protoent
}

// These variables get populated from /etc/protocols and /etc/services
// respectively.
var (
Protocols []*Protoent
Services []*Servent
)

func Init(protocolsFile, servicesFile string) error {
Protocols = []*Protoent{}
Services = []*Servent{}

protoMap := make(map[string]*Protoent)

// Load protocols
if protocolsFile == "" {
protocolsFile = "/etc/protocols"
}
data, err := ioutil.ReadFile(protocolsFile)
if err != nil {
return err
}

for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
split := strings.SplitN(line, "#", 2)
fields := strings.Fields(split[0])
if len(fields) < 2 {
continue
}

num, err := strconv.ParseInt(fields[1], 10, 32)
if err != nil {
return err
}

protoent := &Protoent{
Name: fields[0],
Aliases: fields[2:],
Number: int(num),
}
Protocols = append(Protocols, protoent)

protoMap[fields[0]] = protoent
}

// Load services
if servicesFile == "" {
servicesFile = "/etc/services"
}
data, err = ioutil.ReadFile(servicesFile)
if err != nil {
return err
}

for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
split := strings.SplitN(line, "#", 2)
fields := strings.Fields(split[0])
if len(fields) < 2 {
continue
}

name := fields[0]
portproto := strings.SplitN(fields[1], "/", 2)
port, err := strconv.ParseInt(portproto[0], 10, 32)
if err != nil {
return err
}

proto := portproto[1]
aliases := fields[2:]

Services = append(Services, &Servent{
Name: name,
Aliases: aliases,
Port: int(port),
Protocol: protoMap[proto],
})
}

return nil
}

// Equal checks if two Protoents are the same, which is the case if
// their protocol numbers are identical or when both Protoents are
// nil.
func (this *Protoent) Equal(other *Protoent) bool {
if this == nil && other == nil {
return true
}

if this == nil || other == nil {
return false
}

return this.Number == other.Number
}

// Equal checks if two Servents are the same, which is the case if
// their port numbers and protocols are identical or when both
// Servents are nil.
func (this *Servent) Equal(other *Servent) bool {
if this == nil && other == nil {
return true
}

if this == nil || other == nil {
return false
}

return this.Port == other.Port &&
this.Protocol.Equal(other.Protocol)
}

// GetProtoByNumber returns the Protoent for a given protocol number.
func GetProtoByNumber(num int) (protoent *Protoent) {
for _, protoent := range Protocols {
if protoent.Number == num {
return protoent
}
}
return nil
}

// GetProtoByName returns the Protoent whose name or any of its
// aliases matches the argument.
func GetProtoByName(name string) (protoent *Protoent) {
for _, protoent := range Protocols {
if protoent.Name == name {
return protoent
}

for _, alias := range protoent.Aliases {
if alias == name {
return protoent
}
}
}

return nil
}

// GetServByName returns the Servent for a given service name or alias
// and protocol. If the protocol is nil, the first service matching
// the service name is returned.
func GetServByName(name string, protocol *Protoent) (servent *Servent) {
for _, servent := range Services {
if !servent.Protocol.Equal(protocol) {
continue
}

if servent.Name == name {
return servent
}

for _, alias := range servent.Aliases {
if alias == name {
return servent
}
}
}

return nil
}

// GetServByPort returns the Servent for a given port number and
// protocol. If the protocol is nil, the first service matching the
// port number is returned.
func GetServByPort(port int, protocol *Protoent) *Servent {
for _, servent := range Services {
if servent.Port == port && servent.Protocol.Equal(protocol) {
return servent
}
}

return nil
}
12 changes: 11 additions & 1 deletion pkg/pipeline/transform/transform_network.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ import (
"github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/connection_tracking"
"github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/kubernetes"
"github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/location"
netdb "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/network_services"
log "github.com/sirupsen/logrus"
"honnef.co/go/netdb"
)

type Network struct {
Expand Down Expand Up @@ -168,6 +168,7 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) {
var needToInitLocationDB = false
var needToInitKubeData = false
var needToInitConnectionTracking = false
var needToInitNetworkServices = false

jsonNetworkTransform := params.Transform.Network
for _, rule := range jsonNetworkTransform.Rules {
Expand All @@ -178,6 +179,8 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) {
needToInitKubeData = true
case api.TransformNetworkOperationName("ConnTracking"):
needToInitConnectionTracking = true
case api.TransformNetworkOperationName("AddService"):
needToInitNetworkServices = true
}
}

Expand All @@ -199,6 +202,13 @@ func NewTransformNetwork(params config.StageParam) (Transformer, error) {
}
}

if needToInitNetworkServices {
err := netdb.Init(jsonNetworkTransform.ProtocolsFile, jsonNetworkTransform.ServicesFile)
if err != nil {
return nil, err
}
}

return &Network{
api.TransformNetwork{
Rules: jsonNetworkTransform.Rules,
Expand Down
Loading