From 1b72e8a05ac7817ef0452f350c040a72fb5babc7 Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Tue, 1 Mar 2022 09:40:05 +0100 Subject: [PATCH 1/5] Integrating Goflow-Kube decoration for Netobserv --- go.mod | 6 +- go.sum | 82 ++++++- pkg/api/ingest_collector.go | 5 +- pkg/config/config.go | 12 +- pkg/pipeline/ingest/ingest_collector.go | 71 +++--- pkg/pipeline/ingest/ingest_collector_test.go | 54 +++++ pkg/pipeline/pipeline.go | 32 +-- pkg/pipeline/pipeline_netobs_test.go | 203 ++++++++++++++++++ .../transform/kubernetes/kubernetes.go | 63 +++--- pkg/pipeline/transform/netobserv/enrich.go | 133 ++++++++++++ .../transform/netobserv/enrich_test.go | 131 +++++++++++ pkg/pipeline/transform/netobserv/meta.go | 183 ++++++++++++++++ pkg/pipeline/transform/netobserv/mock.go | 85 ++++++++ pkg/pipeline/transform/transform.go | 7 +- pkg/pipeline/write/write_loki_test.go | 32 +++ pkg/test/eventually.go | 101 +++++++++ pkg/test/eventually_test.go | 58 +++++ pkg/test/ipfix.go | 114 ++++++++++ pkg/test/utils.go | 54 +++++ 19 files changed, 1345 insertions(+), 81 deletions(-) create mode 100644 pkg/pipeline/ingest/ingest_collector_test.go create mode 100644 pkg/pipeline/pipeline_netobs_test.go create mode 100644 pkg/pipeline/transform/netobserv/enrich.go create mode 100644 pkg/pipeline/transform/netobserv/enrich_test.go create mode 100644 pkg/pipeline/transform/netobserv/meta.go create mode 100644 pkg/pipeline/transform/netobserv/mock.go create mode 100644 pkg/test/eventually.go create mode 100644 pkg/test/eventually_test.go create mode 100644 pkg/test/ipfix.go diff --git a/go.mod b/go.mod index b6091ce21..e90271657 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.17 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/go-kit/kit v0.12.0 + github.com/golang/snappy v0.0.4 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb github.com/ip2location/ip2location-go/v9 v9.2.0 github.com/json-iterator/go v1.1.12 @@ -20,6 +21,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.10.1 github.com/stretchr/testify v1.7.0 + github.com/vmware/go-ipfix v0.5.12 golang.org/x/net v0.0.0-20220225172249-27dd8689420f google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v2 v2.4.0 @@ -33,13 +35,13 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-kit/log v0.2.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect @@ -50,7 +52,7 @@ require ( github.com/klauspost/compress v1.13.6 // indirect github.com/libp2p/go-reuseport v0.1.0 // indirect github.com/magiconair/properties v1.8.5 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect diff --git a/go.sum b/go.sum index 19089df5d..75f702ed6 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -56,6 +57,7 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.11.10/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= @@ -91,6 +93,7 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/sarama v1.27.2/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= @@ -125,6 +128,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= @@ -159,21 +163,28 @@ github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/digitalocean/godo v1.46.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -185,6 +196,7 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -205,6 +217,7 @@ github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPO github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -212,8 +225,10 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -245,6 +260,7 @@ github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNV github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -346,6 +362,7 @@ github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGw github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -445,14 +462,18 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.15.0/go.mod h1:vO11I9oWA+KsxmfFQPhLnnIb1VDE24M+pdxZFiuZcA8= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/api v1.7.0/go.mod h1:1NSuaUUkFaJzMasbfq/11wKYWSR67Xn6r2DXKhuDNFg= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.6.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= @@ -477,6 +498,7 @@ github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjG github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -518,6 +540,7 @@ github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mq github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= github.com/ip2location/ip2location-go/v9 v9.2.0 h1:ZWURUspTpe2gxMQGtcQScocBd73gZKUEWlHCFUJYLHk= github.com/ip2location/ip2location-go/v9 v9.2.0/go.mod h1:s5SV6YZL10TpfPpXw//7fEJC65G/yH7Oh+Tjq9JcQEQ= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -551,6 +574,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -579,6 +603,7 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -606,8 +631,9 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -626,6 +652,7 @@ github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -699,6 +726,7 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -708,9 +736,15 @@ github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHu github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pion/dtls/v2 v2.0.3/go.mod h1:TUjyL8bf8LH95h81Xj7kATmzMRt29F/4lxpIPj2Xe4Y= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/transport v0.10.0/go.mod h1:BnHnUipd0rZQyTVB2SBGojFHT9CBt5C5TcsJSQGkvSE= +github.com/pion/transport v0.10.1/go.mod h1:PBis1stIILMiis0PewDw91WJeLJkyIMcEk+DwKOzf4A= +github.com/pion/udp v0.1.0/go.mod h1:BPELIjbwE9PRbd/zxI/KYBnbo7B6+oA6YuEaNE8lths= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -725,6 +759,7 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg github.com/prometheus/alertmanager v0.21.0/go.mod h1:h7tJ81NA0VLWvWEayi1QltevFkLF3KxmC/malTcT8Go= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= @@ -741,7 +776,9 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= @@ -755,16 +792,20 @@ github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuI github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/prometheus v1.8.2-0.20201028100903-3245b3267b24 h1:V/4Cj2GytqdqK7OMEz6c4LNjey3SNyfw3pg5jPKtJvQ= github.com/prometheus/prometheus v1.8.2-0.20201028100903-3245b3267b24/go.mod h1:MDRkz271loM/PrYN+wUNEaTMDGSP760MQzB0yEjdgSQ= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -802,6 +843,7 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= @@ -810,8 +852,10 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.3.0 h1:R7cSvGu+Vv+qX0gW5R/85dx2kmmJT5z5NM8ifdYjdn0= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -819,6 +863,7 @@ github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= @@ -843,12 +888,15 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/vmware/go-ipfix v0.5.12 h1:mqQknlvnvDY25apPNy9c27ri3FMDFIhzvO68Kk5Qp58= +github.com/vmware/go-ipfix v0.5.12/go.mod h1:yzbG1rv+yJ8GeMrRm+MDhOV3akygNZUHLhC1pDoD2AY= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= @@ -863,6 +911,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -884,6 +933,7 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= @@ -916,7 +966,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210915214749-c084706c2272 h1:3erb+vDS8lU1sxfDHF4/hhWyaXnhIaO+7RgL4fDZORA= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1000,12 +1052,14 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -1108,6 +1162,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201008064518-c1f3e3309c71/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1117,6 +1172,7 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1143,7 +1199,9 @@ golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1162,6 +1220,7 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1189,6 +1248,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1197,6 +1257,7 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1420,8 +1481,9 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1429,8 +1491,14 @@ gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1 gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -1450,6 +1518,8 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/netdb v0.0.0-20210921115105-e902e863d85d h1:yjDpoTxoYVpCt04OYp8zlZsKtrEOK1O4U7l2aWbn3D8= honnef.co/go/netdb v0.0.0-20210921115105-e902e863d85d/go.mod h1:rbNo0ST5hSazCG4rGfpHrwnwvzP1QX62WbhzD+ghGzs= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1461,26 +1531,33 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= +k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= k8s.io/api v0.23.4 h1:85gnfXQOWbJa1SiWGpE9EEtHs0UVvDyIsSMpEtl2D4E= k8s.io/api v0.23.4/go.mod h1:i77F4JfyNNrhOjZF7OwwNJS5Y1S9dpwvb9iYRYRczfI= k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= k8s.io/apimachinery v0.23.4 h1:fhnuMd/xUL3Cjfl64j5ULKZ1/J9n8NuQEgNL+WXWfdM= k8s.io/apimachinery v0.23.4/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= +k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA= k8s.io/client-go v0.23.4 h1:YVWvPeerA2gpUudLelvsolzH7c2sFoXXR5wM/sWqNFU= k8s.io/client-go v0.23.4/go.mod h1:PKnIL4pqLuvYUK1WU7RLTMYKPiIh7MYShLshtRY9cj0= +k8s.io/component-base v0.21.0/go.mod h1:qvtjz6X0USWXbgmbfXR+Agik4RZ3jv2Bgr5QnZzdPYw= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1492,6 +1569,7 @@ sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtud sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pkg/api/ingest_collector.go b/pkg/api/ingest_collector.go index fbcc37e28..8c05a6cd3 100644 --- a/pkg/api/ingest_collector.go +++ b/pkg/api/ingest_collector.go @@ -18,6 +18,7 @@ package api type IngestCollector struct { - HostName string `yaml:"hostName" doc:"the hostname to listen on"` - Port int `yaml:"port" doc:"the port number to listen on"` + HostName string `yaml:"hostName" doc:"the hostname to listen on"` + Port int `yaml:"port" doc:"the port number to listen on"` + BatchMaxLen int `yaml:"batchMaxLen" doc:"the number of accumulated flows before being forwarded for processing"` } diff --git a/pkg/config/config.go b/pkg/config/config.go index 8158dcbd4..c8feea853 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -77,9 +77,15 @@ type Decode struct { } type Transform struct { - Type string - Generic api.TransformGeneric - Network api.TransformNetwork + Type string + Generic api.TransformGeneric + Network api.TransformNetwork + KubeEnrich KubeEnrich `json:"k8s_enrich"` +} + +type KubeEnrich struct { + KubeConfigPath string `json:"kubeConfigPath"` + IPFields map[string]string `json:"ipFields"` } type Extract struct { diff --git a/pkg/pipeline/ingest/ingest_collector.go b/pkg/pipeline/ingest/ingest_collector.go index 6268dbf12..8238ea6e7 100644 --- a/pkg/pipeline/ingest/ingest_collector.go +++ b/pkg/pipeline/ingest/ingest_collector.go @@ -25,8 +25,9 @@ import ( "net" "time" + "github.com/netobserv/flowlogs-pipeline/pkg/api" + ms "github.com/mitchellh/mapstructure" - "github.com/netobserv/flowlogs-pipeline/pkg/config" pUtils "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/utils" goflowFormat "github.com/netsampler/goflow2/format" goflowCommonFormat "github.com/netsampler/goflow2/format/common" @@ -37,14 +38,19 @@ import ( "google.golang.org/protobuf/proto" ) -const channelSize = 1000 -const batchMaxTimeInMilliSecs = 1000 +const ( + channelSize = 1000 + defaultBatchFlushTime = time.Second + defaultBatchMaxLength = 500 +) type ingestCollector struct { - hostname string - port int - in chan map[string]interface{} - exitChan chan bool + hostname string + port int + in chan map[string]interface{} + batchFlushTime time.Duration + batchMaxLength int + exitChan chan bool } // TransportWrapper is an implementation of the goflow2 transport interface @@ -104,11 +110,11 @@ func (r *ingestCollector) Ingest(out chan<- []interface{}) { func (r *ingestCollector) initCollectorListener(ctx context.Context) { transporter := NewWrapper(r.in) + formatter, err := goflowFormat.FindFormat(ctx, "pb") + if err != nil { + log.Fatal(err) + } go func() { - formatter, err := goflowFormat.FindFormat(ctx, "pb") - if err != nil { - log.Fatal(err) - } sNF := &utils.StateNetFlow{ Format: formatter, Transport: transporter, @@ -122,10 +128,6 @@ func (r *ingestCollector) initCollectorListener(ctx context.Context) { }() go func() { - formatter, err := goflowFormat.FindFormat(ctx, "pb") - if err != nil { - log.Fatal(err) - } sLegacyNF := &utils.StateNFLegacy{ Format: formatter, Transport: transporter, @@ -141,45 +143,60 @@ func (r *ingestCollector) initCollectorListener(ctx context.Context) { func (r *ingestCollector) processLogLines(out chan<- []interface{}) { var records []interface{} + // Maximum batch time for each batch + flushRecords := time.NewTicker(r.batchFlushTime) + defer flushRecords.Stop() for { select { case <-r.exitChan: log.Debugf("exiting ingestCollector because of signal") return case record := <-r.in: + // TODO: for efficiency, consider forwarding directly as map, + // as this is reverted back from string to map in later pipeline stages recordAsBytes, _ := json.Marshal(record) records = append(records, string(recordAsBytes)) - case <-time.After(time.Millisecond * batchMaxTimeInMilliSecs): // Maximum batch time for each batch + if len(records) >= r.batchMaxLength { + log.Debugf("ingestCollector sending %d entries", len(records)) + out <- records + records = []interface{}{} + } + case <-flushRecords.C: // Process batch of records (if not empty) if len(records) > 0 { log.Debugf("ingestCollector sending %d entries", len(records)) out <- records + records = []interface{}{} } - records = []interface{}{} } } } // NewIngestCollector create a new ingester -func NewIngestCollector(params config.Param) (Ingester, error) { - jsonIngestCollector := params.Ingest.Collector - - if jsonIngestCollector.HostName == "" { +func NewIngestCollector(params api.IngestCollector) (Ingester, error) { + if params.HostName == "" { return nil, fmt.Errorf("ingest hostname not specified") } - if jsonIngestCollector.Port == 0 { + if params.Port == 0 { return nil, fmt.Errorf("ingest port not specified") } - log.Infof("hostname = %s", jsonIngestCollector.HostName) - log.Infof("port = %d", jsonIngestCollector.Port) + log.Infof("hostname = %s", params.HostName) + log.Infof("port = %d", params.Port) ch := make(chan bool, 1) pUtils.RegisterExitChannel(ch) + bml := defaultBatchMaxLength + if params.BatchMaxLen != 0 { + bml = params.BatchMaxLen + } + return &ingestCollector{ - hostname: jsonIngestCollector.HostName, - port: jsonIngestCollector.Port, - exitChan: ch, + hostname: params.HostName, + port: params.Port, + exitChan: ch, + batchFlushTime: defaultBatchFlushTime, + batchMaxLength: bml, }, nil } diff --git a/pkg/pipeline/ingest/ingest_collector_test.go b/pkg/pipeline/ingest/ingest_collector_test.go new file mode 100644 index 000000000..1b0f93130 --- /dev/null +++ b/pkg/pipeline/ingest/ingest_collector_test.go @@ -0,0 +1,54 @@ +package ingest + +import ( + "encoding/json" + "testing" + "time" + + "github.com/netobserv/flowlogs-pipeline/pkg/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const timeout = 5 * time.Second + +func TestIngest(t *testing.T) { + collectorPort, err := test.UDPPort() + require.NoError(t, err) + ic := &ingestCollector{ + hostname: "0.0.0.0", + port: collectorPort, + batchFlushTime: 10 * time.Millisecond, + exitChan: make(chan bool), + } + forwarded := make(chan []interface{}) + defer close(forwarded) + + // GIVEN an IPFIX collector Ingester + go ic.Ingest(forwarded) + + client, err := test.NewIPFIXClient(collectorPort) + require.NoError(t, err) + + // The IPFIX client might send information before the Ingester is actually listening, + // so we might need to repeat the submission + test.Eventually(t, timeout, func(t require.TestingT) { + // WHEN the service receives an IPFIX message + require.NoError(t, client.SendTemplate()) + require.NoError(t, client.SendFlow(12345678, "1.2.3.4")) + + select { + case received := <-forwarded: + // THEN the bytes are forwarded + require.NotEmpty(t, received) + require.IsType(t, "string", received[0]) + flow := map[string]interface{}{} + require.NoError(t, json.Unmarshal([]byte(received[0].(string)), &flow)) + assert.EqualValues(t, 12345678, flow["TimeFlowStart"]) + assert.EqualValues(t, 12345678, flow["TimeFlowEnd"]) + assert.Equal(t, "1.2.3.4", flow["SrcAddr"]) + case <-time.After(50 * time.Millisecond): + require.Fail(t, "error waiting for ingester to forward received data") + } + }) +} diff --git a/pkg/pipeline/pipeline.go b/pkg/pipeline/pipeline.go index 989145ee7..6b0aa186d 100644 --- a/pkg/pipeline/pipeline.go +++ b/pkg/pipeline/pipeline.go @@ -18,8 +18,10 @@ package pipeline import ( + "context" "errors" "fmt" + "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/netobserv" "github.com/heptiolabs/healthcheck" "github.com/netobserv/flowlogs-pipeline/pkg/config" @@ -84,7 +86,7 @@ func getIngester(params config.Param) (ingest.Ingester, error) { case "file_chunks": ingester, err = ingest.NewFileChunks(params) case "collector": - ingester, err = ingest.NewIngestCollector(params) + ingester, err = ingest.NewIngestCollector(params.Ingest.Collector) case "kafka": ingester, err = ingest.NewIngestKafka(params) default: @@ -135,6 +137,8 @@ func getTransformer(params config.Param) (transform.Transformer, error) { transformer, err = transform.NewTransformNetwork(params) case transform.OperationNone: transformer, err = transform.NewTransformNone() + case transform.OperationK8sEnrich: + transformer, err = netobserv.StartEnricher(context.TODO(), params.Transform.KubeEnrich) default: panic(fmt.Sprintf("`transform` type %s not defined; if no encoder needed, specify `none`", params.Transform.Type)) } @@ -213,7 +217,11 @@ func NewPipeline() (*Pipeline, error) { configParams := config.Parameters log.Debugf("configParams = %v ", configParams) - return newBuilder(configParams, stages).build() + build := newBuilder(configParams, stages) + if err := build.readStages(); err != nil { + return nil, err + } + return build.build() } func newBuilder(params []config.Param, stages []config.Stage) *builder { @@ -268,11 +276,8 @@ func (b *builder) readStages() error { } // reads the configured Go stages and connects between them +// readStages must be invoked before this func (b *builder) build() (*Pipeline, error) { - if err := b.readStages(); err != nil { - return nil, err - } - for _, connection := range b.configStages { if connection.Name == "" || connection.Follows == "" { // ignore entries that do not represent a connection @@ -283,7 +288,7 @@ func (b *builder) build() (*Pipeline, error) { if !ok { return nil, fmt.Errorf("unknown pipeline stage: %s", connection.Name) } - dstNode, err := b.getStageNode(dstEntry) + dstNode, err := b.getStageNode(dstEntry, connection.Name) if err != nil { return nil, err } @@ -297,14 +302,14 @@ func (b *builder) build() (*Pipeline, error) { if !ok { return nil, fmt.Errorf("unknown pipeline stage: %s", connection.Follows) } - srcNode, err := b.getStageNode(srcEntry) + srcNode, err := b.getStageNode(srcEntry, connection.Follows) if err != nil { return nil, err } src, ok := srcNode.(node.Sender) if !ok { return nil, fmt.Errorf("stage %q of type %q can't send data", - connection.Name, dstEntry.stageType) + connection.Follows, srcEntry.stageType) } log.Debugf("connecting stages: %s --> %s", connection.Follows, connection.Name) @@ -337,8 +342,8 @@ func (b *builder) build() (*Pipeline, error) { }, nil } -func (b *builder) getStageNode(pe *pipelineEntry) (interface{}, error) { - if stg, ok := b.createdStages[pe.configStage.Name]; ok { +func (b *builder) getStageNode(pe *pipelineEntry, stageID string) (interface{}, error) { + if stg, ok := b.createdStages[stageID]; ok { return stg, nil } var stage interface{} @@ -384,17 +389,18 @@ func (b *builder) getStageNode(pe *pipelineEntry) (interface{}, error) { default: return nil, fmt.Errorf("invalid stage type: %s", pe.stageType) } - b.createdStages[pe.configStage.Name] = stage + b.createdStages[stageID] = stage return stage, nil } func (p *Pipeline) Run() { - p.IsRunning = true // starting the graph for _, s := range p.startNodes { s.Start() } + p.IsRunning = true + // blocking the execution until the graph terminal stages end for _, t := range p.terminalNodes { <-t.Done() diff --git a/pkg/pipeline/pipeline_netobs_test.go b/pkg/pipeline/pipeline_netobs_test.go new file mode 100644 index 000000000..ad8100c36 --- /dev/null +++ b/pkg/pipeline/pipeline_netobs_test.go @@ -0,0 +1,203 @@ +package pipeline + +import ( + "fmt" + "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/decode" + "net/http/httptest" + "testing" + "time" + + "github.com/netobserv/flowlogs-pipeline/pkg/api" + "github.com/netobserv/flowlogs-pipeline/pkg/config" + "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/ingest" + "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/netobserv" + "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/write" + "github.com/netobserv/flowlogs-pipeline/pkg/test" + "github.com/prometheus/common/model" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// Tests in this file are aimed at validating the Flowlogs Pipeline configuration for its +// usage within the network observability kube enrichment + +const timeout = 5000 * time.Second + +type testPipeline struct { + pipe *Pipeline + client *test.IPFIXClient + lokiFlows chan map[string]interface{} + fakeloki *httptest.Server + stopInformers chan struct{} +} + +func (p *testPipeline) Close() { + close(p.lokiFlows) + close(p.stopInformers) + p.fakeloki.Close() +} + +// NetobsTestPipeline reproduces a basic network observability pipeline and returns it. +// It is not created from the global YAML text configuration because +// we need a programmatic way to instantiate a pipeline and inject there the informers +// and the fake Loki listener backend +func NetobsTestPipeline(t *testing.T) *testPipeline { + // STAGE 1: IPFIX ingester + // Find a free port for the IPFIX collector + ipfixPort, err := test.UDPPort() + require.NoError(t, err) + collector, err := ingest.NewIngestCollector(api.IngestCollector{ + HostName: "0.0.0.0", + Port: ipfixPort, + BatchMaxLen: 1, + }) + require.NoError(t, err) + + // STAGE 2: decoding from flow's JSON data to GenericMAP + decoder, err := decode.NewDecodeJson() + require.NoError(t, err) + + // STAGE 3: Kube Enricher getting its data from a mocked informer + informers := netobserv.NewInformers(fake.NewSimpleClientset(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "influxdb-v2", + Namespace: "default", + Annotations: map[string]string{ + "anonotation": "true", + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + PodIP: "1.2.3.4", + PodIPs: []corev1.PodIP{{IP: "1.2.3.4"}}, + }, + })) + stopInformers := make(chan struct{}) + require.NoError(t, informers.Start(stopInformers)) + enricher := &netobserv.Enricher{ + Config: config.KubeEnrich{ + IPFields: map[string]string{"SrcAddr": ""}, + }, + Informers: informers, + } + + // STAGE 4: Loki exporter writing to a fake loki server that captures all the forwarded data + lokiFlows := make(chan map[string]interface{}, 256) + fakeLoki := httptest.NewServer(test.FakeLokiHandler(lokiFlows)) + log.Debugf("fake loki URL: %s", fakeLoki.URL) + lokiConfig := config.Param{ + Name: "loki", + Write: config.Write{Loki: api.WriteLoki{ + URL: fakeLoki.URL, + TenantID: "foo", + BatchWait: "1s", + BatchSize: 1, + Timeout: "1s", + StaticLabels: map[model.LabelName]model.LabelValue{ + "testApp": "network-observability-test", + }, + }}, + } + // NewWriteLoki requires this to not fail during initialization. + // TODO: simplify configuration and make not depending it on global variables + config.Opt.Parameters = fmt.Sprintf( + `[{"name":"write1","write":{"type":"loki","loki":{"url":"%s","batchSize":1,`+ + `"staticLabels":{"testApp":"network-observability-test"}}}}]`, fakeLoki.URL) + config.Parameters = []config.Param{lokiConfig} + lokiExporter, err := write.NewWriteLoki(lokiConfig) + require.NoError(t, err) + + // Injecting all the components into a builder object, this way + // we can provide test components + pipe, err := (&builder{ + createdStages: map[string]interface{}{}, + pipelineEntryMap: map[string]*pipelineEntry{ + "ingest": {stageType: StageIngest, Ingester: collector}, + "decode": {stageType: StageDecode, Decoder: decoder}, + "enrich": {stageType: StageTransform, Transformer: enricher}, + "loki": {stageType: StageWrite, Writer: lokiExporter}, + }, + configStages: []config.Stage{ + {Name: "loki", Follows: "enrich"}, + {Name: "enrich", Follows: "decode"}, + {Name: "decode", Follows: "ingest"}, + }, + }).build() + require.NoError(t, err) + + go pipe.Run() + + client, err := test.NewIPFIXClient(ipfixPort) + require.NoError(t, err) + return &testPipeline{ + pipe: pipe, + client: client, + lokiFlows: lokiFlows, + fakeloki: fakeLoki, + stopInformers: stopInformers, + } +} + +func TestNetobservEndToEnd(t *testing.T) { + //log.StandardLogger().SetLevel(log.DebugLevel) + // GIVEN a network observability pipeline + pipe := NetobsTestPipeline(t) + defer pipe.Close() + + var flow map[string]interface{} + // Since the collector starts in background, it might take some time to accept a + // flow, so we repeat the first submission until it succeeds + test.Eventually(t, timeout, func(t require.TestingT) { + require.NoError(t, pipe.client.SendTemplate()) + + // WHEN a Pod flow is captured + require.NoError(t, pipe.client.SendFlow(12345678, "1.2.3.4")) + + // THEN the flow data is forwarded to Loki + select { + case flow = <-pipe.lokiFlows: + return + case <-time.After(100 * time.Millisecond): + require.Fail(t, "timeout while waiting for flows") + } + }) + + assert.Equal(t, "1.2.3.4", flow["SrcAddr"]) + assert.EqualValues(t, 12345678, flow["TimeFlowStart"]) + assert.EqualValues(t, 12345678, flow["TimeFlowEnd"]) + + // AND it is decorated with the proper Pod metadata + assert.Equal(t, "influxdb-v2", flow["Pod"]) + assert.Equal(t, "influxdb-v2", flow["Workload"]) + assert.Equal(t, "Pod", flow["WorkloadKind"]) + assert.Equal(t, "default", flow["Namespace"]) + + // AND WHEN a Flow is captured from an unknown Kubernetes entity + require.NoError(t, pipe.client.SendFlow(12345699, "4.3.2.1")) + + // Some flows from the previous resubmissions might still be in queue. We wait until + // the last flow is received + test.Eventually(t, timeout, func(t require.TestingT) { + select { + case flow = <-pipe.lokiFlows: + // THEN the flow data is forwarded anyway + assert.Equal(t, "4.3.2.1", flow["SrcAddr"]) + case <-time.After(timeout): + require.Fail(t, "timeout while waiting for flows") + } + }) + + // THEN it is forwarded anyway + assert.EqualValues(t, 12345699, flow["TimeFlowStart"]) + assert.EqualValues(t, 12345699, flow["TimeFlowEnd"]) + + // BUT without any metadata decoration + assert.NotContains(t, flow, "Pod") + assert.NotContains(t, flow, "Workload") + assert.NotContains(t, flow, "WorkloadKind") + assert.NotContains(t, flow, "Namespace") +} diff --git a/pkg/pipeline/transform/kubernetes/kubernetes.go b/pkg/pipeline/transform/kubernetes/kubernetes.go index 8a69d7c49..ff294ba2c 100644 --- a/pkg/pipeline/transform/kubernetes/kubernetes.go +++ b/pkg/pipeline/transform/kubernetes/kubernetes.go @@ -19,6 +19,10 @@ package kubernetes import ( "fmt" + "net" + "os" + "time" + log "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -28,9 +32,6 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" - "net" - "os" - "time" ) var Data KubeData @@ -201,36 +202,13 @@ func (k *KubeData) NewReplicaSetInformer(informerFactory informers.SharedInforme } func (k *KubeData) InitFromConfig(kubeConfigPath string) error { - var config *rest.Config - var err error - // Initialization variables k.stopChan = make(chan struct{}) k.ipInformers = map[string]cache.SharedIndexInformer{} - if kubeConfigPath != "" { - config, err = clientcmd.BuildConfigFromFlags("", kubeConfigPath) - if err != nil { - return fmt.Errorf("can't build config from %s", kubeConfigPath) - } - } else { - kubeConfigPath = os.Getenv(kubeConfigEnvVariable) - if kubeConfigPath != "" { - config, err = clientcmd.BuildConfigFromFlags("", kubeConfigPath) - if err != nil { - return fmt.Errorf("can't build config from %s", kubeConfigPath) - } - } else { - homeDir, _ := os.UserHomeDir() - config, err = clientcmd.BuildConfigFromFlags("", homeDir+"/.kube/config") - if err != nil { - // creates the in-cluster config - config, err = rest.InClusterConfig() - if err != nil { - return fmt.Errorf("can't access kubenetes. Tried using config from: config parameter, %s env, homedir and InClusterConfig", kubeConfigEnvVariable) - } - } - } + config, err := LoadConfig(kubeConfigPath) + if err != nil { + return err } kubeClient, err := kubernetes.NewForConfig(config) @@ -246,6 +224,33 @@ func (k *KubeData) InitFromConfig(kubeConfigPath string) error { return nil } +func LoadConfig(kubeConfigPath string) (*rest.Config, error) { + // if no config path is provided, load it from the env variable + if kubeConfigPath == "" { + kubeConfigPath = os.Getenv(kubeConfigEnvVariable) + } + // otherwise, load it from the $HOME/.kube/config file + if kubeConfigPath == "" { + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, fmt.Errorf("can't get user home dir: %w", err) + } + kubeConfigPath = homeDir + "/.kube/config" + } + config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) + if err == nil { + return config, nil + } + // fallback: use in-cluster config + config, err = rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("can't access kubenetes. Tried using config from: "+ + "config parameter, %s env, homedir and InClusterConfig. Got: %w", + kubeConfigEnvVariable, err) + } + return config, nil +} + func (k *KubeData) initInformers(client kubernetes.Interface) error { informerFactory := informers.NewSharedInformerFactory(client, syncTime) err := k.NewNodeInformer(informerFactory) diff --git a/pkg/pipeline/transform/netobserv/enrich.go b/pkg/pipeline/transform/netobserv/enrich.go new file mode 100644 index 000000000..eef06615a --- /dev/null +++ b/pkg/pipeline/transform/netobserv/enrich.go @@ -0,0 +1,133 @@ +// Package netobserv implements transformations for Network Observability +package netobserv + +import ( + "context" + "fmt" + "strings" + + kube "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/kubernetes" + "k8s.io/client-go/kubernetes" + + "github.com/netobserv/flowlogs-pipeline/pkg/config" + + "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var elog = logrus.WithField("module", "reader/Enricher") + +var ownerNameFunc = func(owners interface{}, idx int) string { + owner := owners.([]metav1.OwnerReference)[idx] + return owner.Kind + "/" + owner.Name +} + +type Enricher struct { + Config config.KubeEnrich + Informers InformersInterface +} + +func StartEnricher(ctx context.Context, cfg config.KubeEnrich) (*Enricher, error) { + kubeConfig, err := kube.LoadConfig(cfg.KubeConfigPath) + if err != nil { + return nil, err + } + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + return nil, err + } + informers := NewInformers(kubeClient) + if err := informers.Start(ctx.Done()); err != nil { + return nil, fmt.Errorf("can't start informers: %w", err) + } + informers.WaitForCacheSync(ctx.Done()) + return &Enricher{ + Config: cfg, + Informers: informers, + }, nil +} + +func (e *Enricher) Transform(record config.GenericMap) config.GenericMap { + for ipField, prefixOut := range e.Config.IPFields { + val, ok := record[ipField] + if !ok { + elog.Infof("Field %s not found in record", ipField) + continue + } + ip, ok := val.(string) + if !ok { + elog.Warnf("String expected for field %s value %v", ipField, val) + continue + } + if pod := e.Informers.PodByIP(ip); pod != nil { + e.enrichPod(record, prefixOut, pod) + } else { + // If there is no Pod for such IP, we try searching for a service + e.enrichService(ip, record, prefixOut) + } + } + return record +} + +func (e *Enricher) enrichService(ip string, record map[string]interface{}, prefixOut string) { + if svc := e.Informers.ServiceByIP(ip); svc != nil { + fillWorkloadRecord(record, prefixOut, "Service", svc.Name, svc.Namespace) + } else { + elog.Warnf("Failed to get Service [ip=%v]", ip) + } +} + +func (e *Enricher) enrichPod(record map[string]interface{}, prefixOut string, pod *v1.Pod) { + fillPodRecord(record, prefixOut, pod) + var warnings []string + if len(pod.OwnerReferences) > 0 { + warnings = e.checkTooMany(warnings, "owners", "pod "+pod.Name, pod.OwnerReferences, len(pod.OwnerReferences), ownerNameFunc) + ref := pod.OwnerReferences[0] + if ref.Kind == "ReplicaSet" { + // Search deeper (e.g. Deployment, DeploymentConfig) + if rs := e.Informers.ReplicaSet(pod.Namespace, ref.Name); rs != nil { + if len(rs.OwnerReferences) > 0 { + warnings = e.checkTooMany(warnings, "owners", "replica "+rs.Name, rs.OwnerReferences, len(rs.OwnerReferences), ownerNameFunc) + ref = rs.OwnerReferences[0] + } + } else { + elog.Warnf("Failed to get ReplicaSet [ns=%s,name=%s]", pod.Namespace, ref.Name) + } + } + fillWorkloadRecord(record, prefixOut, ref.Kind, ref.Name, "") + } else { + // Consider a pod without owner as self-owned + fillWorkloadRecord(record, prefixOut, "Pod", pod.Name, "") + } + if len(warnings) > 0 { + record[prefixOut+"Warn"] = strings.Join(warnings, "; ") + } +} + +func (e *Enricher) checkTooMany(warnings []string, kind, ref string, items interface{}, size int, nameFunc func(interface{}, int) string) []string { + if size > 1 { + var names []string + for i := 0; i < size; i++ { + names = append(names, nameFunc(items, i)) + } + elog.Tracef("%d %s found for %s, using first", size, kind, ref) + warn := fmt.Sprintf("Several %s found for %s: %s", kind, ref, strings.Join(names, ",")) + warnings = append(warnings, warn) + } + return warnings +} + +func fillPodRecord(record map[string]interface{}, prefix string, pod *v1.Pod) { + record[prefix+"Pod"] = pod.Name + record[prefix+"Namespace"] = pod.Namespace + record[prefix+"HostIP"] = pod.Status.HostIP +} + +func fillWorkloadRecord(record map[string]interface{}, prefix, kind, name, ns string) { + record[prefix+"Workload"] = name + record[prefix+"WorkloadKind"] = kind + if ns != "" { + record[prefix+"Namespace"] = ns + } +} diff --git a/pkg/pipeline/transform/netobserv/enrich_test.go b/pkg/pipeline/transform/netobserv/enrich_test.go new file mode 100644 index 000000000..0bb853ce4 --- /dev/null +++ b/pkg/pipeline/transform/netobserv/enrich_test.go @@ -0,0 +1,131 @@ +package netobserv + +import ( + "testing" + + "github.com/netobserv/flowlogs-pipeline/pkg/config" + "github.com/stretchr/testify/assert" +) + +func setupSimpleEnricher() (*Enricher, *InformersMock) { + informers := new(InformersMock) + r := Enricher{ + Config: config.KubeEnrich{ + IPFields: map[string]string{ + "SrcAddr": "Src", + "DstAddr": "Dst", + }, + }, + Informers: informers, + } + return &r, informers +} + +func TestEnrichNoMatch(t *testing.T) { + assert := assert.New(t) + r, informers := setupSimpleEnricher() + + informers.MockPod("test-pod1", "test-namespace", "10.0.0.1", "10.0.0.100") + informers.MockNoMatch("10.0.0.2") + + records := map[string]interface{}{ + "SrcAddr": "10.0.0.1", + "DstAddr": "10.0.0.2", + } + + records = r.Transform(records) + assert.Equal(map[string]interface{}{ + "SrcAddr": "10.0.0.1", + "SrcPod": "test-pod1", + "SrcNamespace": "test-namespace", + "SrcHostIP": "10.0.0.100", + "SrcWorkload": "test-pod1", + "SrcWorkloadKind": "Pod", + "DstAddr": "10.0.0.2", + }, records) +} + +func TestEnrichSinglePods(t *testing.T) { + assert := assert.New(t) + r, informers := setupSimpleEnricher() + + informers.MockPod("test-pod1", "test-namespace", "10.0.0.1", "10.0.0.100") + informers.MockPod("test-pod2", "test-namespace", "10.0.0.2", "10.0.0.100") + + records := map[string]interface{}{ + "SrcAddr": "10.0.0.1", + "DstAddr": "10.0.0.2", + } + + records = r.Transform(records) + assert.Equal(map[string]interface{}{ + "SrcAddr": "10.0.0.1", + "SrcPod": "test-pod1", + "SrcNamespace": "test-namespace", + "SrcHostIP": "10.0.0.100", + "SrcWorkload": "test-pod1", + "SrcWorkloadKind": "Pod", + "DstAddr": "10.0.0.2", + "DstPod": "test-pod2", + "DstNamespace": "test-namespace", + "DstHostIP": "10.0.0.100", + "DstWorkload": "test-pod2", + "DstWorkloadKind": "Pod", + }, records) +} + +func TestEnrichDeploymentPods(t *testing.T) { + assert := assert.New(t) + r, informers := setupSimpleEnricher() + + informers.MockPodInDepl("test-pod1", "test-namespace", "10.0.0.1", "10.0.0.100", "test-rs-1", "test-deployment1") + informers.MockPodInDepl("test-pod2", "test-namespace", "10.0.0.2", "10.0.0.100", "test-rs-2", "test-deployment2") + + records := map[string]interface{}{ + "SrcAddr": "10.0.0.1", + "DstAddr": "10.0.0.2", + } + + records = r.Transform(records) + assert.Equal(map[string]interface{}{ + "SrcAddr": "10.0.0.1", + "SrcPod": "test-pod1", + "SrcNamespace": "test-namespace", + "SrcHostIP": "10.0.0.100", + "SrcWorkload": "test-deployment1", + "SrcWorkloadKind": "Deployment", + "DstAddr": "10.0.0.2", + "DstPod": "test-pod2", + "DstNamespace": "test-namespace", + "DstHostIP": "10.0.0.100", + "DstWorkload": "test-deployment2", + "DstWorkloadKind": "Deployment", + }, records) +} + +func TestEnrichPodAndService(t *testing.T) { + assert := assert.New(t) + r, informers := setupSimpleEnricher() + + informers.MockPod("test-pod1", "test-namespace", "10.0.0.1", "10.0.0.100") + informers.MockService("test-service", "test-namespace", "10.0.0.2") + + records := map[string]interface{}{ + "SrcAddr": "10.0.0.1", + "DstAddr": "10.0.0.2", + } + + records = r.Transform(records) + assert.Equal(map[string]interface{}{ + "SrcAddr": "10.0.0.1", + "SrcPod": "test-pod1", + "SrcNamespace": "test-namespace", + "SrcHostIP": "10.0.0.100", + "SrcWorkload": "test-pod1", + "SrcWorkloadKind": "Pod", + "DstAddr": "10.0.0.2", + "DstNamespace": "test-namespace", + "DstWorkload": "test-service", + "DstWorkloadKind": "Service", + }, records) +} diff --git a/pkg/pipeline/transform/netobserv/meta.go b/pkg/pipeline/transform/netobserv/meta.go new file mode 100644 index 000000000..df89d5c4a --- /dev/null +++ b/pkg/pipeline/transform/netobserv/meta.go @@ -0,0 +1,183 @@ +package netobserv + +import ( + "fmt" + "io" + "strings" + "time" + + "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +const ( + // NamespaceSeparator used by the K8s informers library to create indices + // that are composed as namespace/name + NamespaceSeparator = "/" + IndexIP = "IP" +) + +var ilog = logrus.WithFields(logrus.Fields{ + "component": fmt.Sprintf("%T", Informers{}), +}) + +type InformersInterface interface { + PodByIP(ip string) *corev1.Pod + ServiceByIP(ip string) *corev1.Service + ReplicaSet(namespace, name string) *appsv1.ReplicaSet +} + +// Informers access for network observability metric decoration +type Informers struct { + InformersInterface + informerFactory informers.SharedInformerFactory + pods cache.SharedIndexInformer + services cache.SharedIndexInformer + replicaSet cache.SharedIndexInformer +} + +func NewInformers(client kubernetes.Interface) *Informers { + // TODO: configure resync time + factory := informers.NewSharedInformerFactory(client, 1*time.Hour) + pods := factory.Core().V1().Pods().Informer() + if err := pods.AddIndexers(map[string]cache.IndexFunc{ + IndexIP: func(obj interface{}) ([]string, error) { + pod := obj.(*corev1.Pod) + ips := make([]string, 0, len(pod.Status.PodIPs)) + for _, ip := range pod.Status.PodIPs { + // ignoring host-networked Pod IPs + if ip.IP != pod.Status.HostIP { + ips = append(ips, ip.IP) + } + } + return ips, nil + }, + }); err != nil { + // this should never happen, as it only returns error if the informer has + // been alrady started + panic(err) + } + services := factory.Core().V1().Services().Informer() + if err := services.AddIndexers(map[string]cache.IndexFunc{ + IndexIP: func(obj interface{}) ([]string, error) { + spec := obj.(*corev1.Service).Spec + if spec.ClusterIP == corev1.ClusterIPNone { + return []string{}, nil + } + return spec.ClusterIPs, nil + }, + }); err != nil { + panic(err) + } + return &Informers{ + informerFactory: factory, + pods: pods, + services: services, + replicaSet: factory.Apps().V1().ReplicaSets().Informer(), + } +} + +func (i *Informers) Start(stopCh <-chan struct{}) error { + i.informerFactory.Start(stopCh) + return nil +} + +func (i *Informers) WaitForCacheSync(stopCh <-chan struct{}) { + i.informerFactory.WaitForCacheSync(stopCh) +} + +func (i *Informers) PodByIP(ip string) *corev1.Pod { + item, err := i.pods.GetIndexer().ByIndex(IndexIP, ip) + if err != nil { + // should never happen as long as we provide the correct index function + // otherwise it's a bug in our code + panic(err) + } + // our provided indexers only return a key, so it's safe to assume 0<=len()<=1 + if len(item) == 0 { + // not found + return nil + } + // since we are excluding host-networked pods, the relation IP:Pod should be 1:1. + if len(item) > 1 { + ilog.WithFields(logrus.Fields{ + "ip": ip, + "results": len(item), + }).Warn("multiple pods for a single IP. Returning the first pod and ignoring the rest") + } + return item[0].(*corev1.Pod) +} + +func (i *Informers) ServiceByIP(ip string) *corev1.Service { + item, err := i.services.GetIndexer().ByIndex(IndexIP, ip) + if err != nil { + // should never happen as long as we provide the correct index function + // otherwise it's a bug in our code + panic(err) + } + if len(item) == 0 { + // not found + return nil + } + // we assume a 1:1 relation between Service and ClusterIP + if len(item) > 1 { + ilog.WithFields(logrus.Fields{ + "ip": ip, + "results": len(item), + }).Warn("multiple services for a single IP. Returning the first service and ignoring the rest") + } + return item[0].(*corev1.Service) +} + +func (i *Informers) ReplicaSet(namespace, name string) *appsv1.ReplicaSet { + item, ok, err := i.replicaSet.GetIndexer().GetByKey(namespace + NamespaceSeparator + name) + if err != nil { + // should never happen. Otherwise it's a bug in our code + panic(err) + } + if !ok { + return nil + } + return item.(*appsv1.ReplicaSet) +} + +func (i *Informers) DebugInfo(out io.Writer) { + fmt.Fprintln(out, "==== Services") + for _, svc := range i.services.GetStore().ListKeys() { + fmt.Fprintln(out, "-", svc) + } + fmt.Fprintln(out, "==== ReplicaSets") + for _, rs := range i.replicaSet.GetStore().ListKeys() { + rskeys := strings.Split(rs, NamespaceSeparator) + rset := i.ReplicaSet(rskeys[0], rskeys[1]) + fmt.Fprintln(out, "-", rs, "replicas:", rset.Status.Replicas) + } + fmt.Fprintln(out, "==== Pods") + for _, pod := range i.pods.GetStore().ListKeys() { + fmt.Fprintln(out, "-", pod) + } + fmt.Fprintln(out, "=== Pods by IP") + for _, ip := range i.pods.GetIndexer().ListIndexFuncValues(IndexIP) { + pod := i.PodByIP(ip) + if pod.Status.PodIP != ip { + panic("ips not equal") + } + fmt.Fprintln(out, "-", ip, ":", pod.Name) + } + fmt.Fprintln(out, "=== Services by IP") + for _, ip := range i.services.GetIndexer().ListIndexFuncValues(IndexIP) { + svc := i.ServiceByIP(ip) + if svc != nil { + if svc.Spec.ClusterIP != ip { + panic("ips not equal") + } + fmt.Fprintln(out, "-", ip, ":", svc.Name) + } else { + fmt.Fprintln(out, "-", ip, "not found in index") + } + } +} diff --git a/pkg/pipeline/transform/netobserv/mock.go b/pkg/pipeline/transform/netobserv/mock.go new file mode 100644 index 000000000..5ac535fea --- /dev/null +++ b/pkg/pipeline/transform/netobserv/mock.go @@ -0,0 +1,85 @@ +package netobserv + +import ( + "github.com/stretchr/testify/mock" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// InformersMock provides an informers' implementation for unit testing +type InformersMock struct { + mock.Mock + InformersInterface +} + +func (o *InformersMock) PodByIP(ip string) *corev1.Pod { + args := o.Called(ip) + return args.Get(0).(*corev1.Pod) +} + +func (o *InformersMock) ServiceByIP(ip string) *corev1.Service { + args := o.Called(ip) + return args.Get(0).(*corev1.Service) +} + +func (o *InformersMock) ReplicaSet(namespace, name string) *appsv1.ReplicaSet { + args := o.Called(namespace, name) + return args.Get(0).(*appsv1.ReplicaSet) +} + +func fakePod(name, ns, host string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Status: corev1.PodStatus{ + HostIP: host, + }, + } +} + +func fakeService(name, ns string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + } +} + +func (o *InformersMock) MockPod(name, ns, ip, host string) { + o.On("PodByIP", ip).Return(fakePod(name, ns, host)) +} + +func (o *InformersMock) MockPodInDepl(name, ns, ip, host, rs, depl string) { + pod := fakePod(name, ns, host) + pod.OwnerReferences = append(pod.OwnerReferences, metav1.OwnerReference{ + Name: rs, + Kind: "ReplicaSet", + }) + o.On("PodByIP", ip).Return(pod) + o.On("ReplicaSet", ns, rs).Return(&appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: rs, + Namespace: ns, + OwnerReferences: []metav1.OwnerReference{ + { + Name: depl, + Kind: "Deployment", + }, + }, + }, + }) +} + +func (o *InformersMock) MockService(name, ns, ip string) { + o.On("PodByIP", ip).Return((*corev1.Pod)(nil)) + o.On("ServiceByIP", ip).Return(fakeService(name, ns)) +} + +func (o *InformersMock) MockNoMatch(ip string) { + o.On("PodByIP", ip).Return((*corev1.Pod)(nil)) + o.On("ServiceByIP", ip).Return((*corev1.Service)(nil)) +} diff --git a/pkg/pipeline/transform/transform.go b/pkg/pipeline/transform/transform.go index a92725375..f536aa713 100644 --- a/pkg/pipeline/transform/transform.go +++ b/pkg/pipeline/transform/transform.go @@ -50,9 +50,10 @@ type Definition struct { type Definitions []Definition const ( - OperationGeneric = "generic" - OperationNetwork = "network" - OperationNone = "none" + OperationGeneric = "generic" + OperationNetwork = "network" + OperationK8sEnrich = "k8sEnrich" + OperationNone = "none" ) func ExecuteTransform(transformer Transformer, in []config.GenericMap) []config.GenericMap { diff --git a/pkg/pipeline/write/write_loki_test.go b/pkg/pipeline/write/write_loki_test.go index 820f71558..699020051 100644 --- a/pkg/pipeline/write/write_loki_test.go +++ b/pkg/pipeline/write/write_loki_test.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/netobserv/flowlogs-pipeline/pkg/config" "github.com/netobserv/flowlogs-pipeline/pkg/test" + "net/http/httptest" "testing" "time" @@ -31,6 +32,8 @@ import ( "github.com/stretchr/testify/require" ) +const timeout = 5 * time.Second + type fakeEmitter struct { mock.Mock } @@ -249,3 +252,32 @@ parameters: "ba_z": "isBaz", }, mock.Anything, mock.Anything) } + +func TestHTTPInvocations(t *testing.T) { + lokiFlows := make(chan map[string]interface{}, 256) + fakeLoki := httptest.NewServer(test.FakeLokiHandler(lokiFlows)) + + var yamlConfig = fmt.Sprintf(` +log-level: debug +pipeline: + - name: write1 +parameters: + - name: write1 + write: + type: loki + loki: + url: %s +`, fakeLoki.URL) + test.InitConfig(t, yamlConfig) + loki, err := NewWriteLoki(config.Parameters[0]) + require.NoError(t, err) + + require.NoError(t, loki.ProcessRecord(map[string]interface{}{"foo": "bar", "baz": "bae"})) + + select { + case flow := <-lokiFlows: + assert.Equal(t, map[string]interface{}{"foo": "bar", "baz": "bae"}, flow) + case <-time.After(timeout): + require.Fail(t, "timeout while waiting for LokiWriter to forward data") + } +} diff --git a/pkg/test/eventually.go b/pkg/test/eventually.go new file mode 100644 index 000000000..a117553ea --- /dev/null +++ b/pkg/test/eventually.go @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2022 IBM, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// Eventually retries a test until it eventually succeeds. If the timeout is reached, the test fails +// with the same failure as its last execution. +func Eventually(t *testing.T, timeout time.Duration, testFunc func(_ require.TestingT)) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + success := make(chan interface{}) + errorCh := make(chan error) + failCh := make(chan error) + + go func() { + for ctx.Err() == nil { + result := testResult{failed: false, errorCh: errorCh, failCh: failCh} + // Executing the function to test + testFunc(&result) + // If the function didn't reported failure and didn't reached timeout + if !result.HasFailed() && ctx.Err() == nil { + success <- 1 + break + } + } + }() + + // Wait for success or timeout + var err, fail error + for { + select { + case <-success: + return + case err = <-errorCh: + case fail = <-failCh: + case <-ctx.Done(): + if err != nil { + t.Error(err) + } else if fail != nil { + t.Error(fail) + } else { + t.Error("timeout while waiting for test to complete") + } + return + } + } +} + +// util class for Eventually +type testResult struct { + sync.RWMutex + failed bool + errorCh chan<- error + failCh chan<- error +} + +func (te *testResult) Errorf(format string, args ...interface{}) { + te.Lock() + te.failed = true + te.Unlock() + te.errorCh <- fmt.Errorf(format, args...) +} + +func (te *testResult) FailNow() { + te.Lock() + te.failed = true + te.Unlock() + te.failCh <- errors.New("test failed") +} + +func (te *testResult) HasFailed() bool { + te.RLock() + defer te.RUnlock() + return te.failed +} diff --git a/pkg/test/eventually_test.go b/pkg/test/eventually_test.go new file mode 100644 index 000000000..6a23ec815 --- /dev/null +++ b/pkg/test/eventually_test.go @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2022 IBM, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEventually_Error(t *testing.T) { + innerTest := &testing.T{} + Eventually(innerTest, 10*time.Millisecond, func(t require.TestingT) { + require.True(t, false) + }) + assert.True(t, innerTest.Failed()) +} + +func TestEventually_Fail(t *testing.T) { + innerTest := &testing.T{} + Eventually(innerTest, 10*time.Millisecond, func(t require.TestingT) { + t.FailNow() + }) + assert.True(t, innerTest.Failed()) +} + +func TestEventually_Timeout(t *testing.T) { + innerTest := &testing.T{} + Eventually(innerTest, 10*time.Millisecond, func(t require.TestingT) { + time.Sleep(5 * time.Second) + }) + assert.True(t, innerTest.Failed()) +} + +func TestEventually_Success(t *testing.T) { + num := 3 + Eventually(t, 5*time.Second, func(t require.TestingT) { + require.Equal(t, 0, num) + num-- + }) +} diff --git a/pkg/test/ipfix.go b/pkg/test/ipfix.go new file mode 100644 index 000000000..9610b8eac --- /dev/null +++ b/pkg/test/ipfix.go @@ -0,0 +1,114 @@ +package test + +import ( + "fmt" + "net" + "time" + + "github.com/vmware/go-ipfix/pkg/entities" +) + +const ( + templateID = 256 + observationDomainID = 1 +) + +// Values taken from https://www.iana.org/assignments/ipfix/ipfix.xhtml +var ( + sourceIPv4Address = entities.NewInfoElement("sourceIPv4Address", 8, entities.Ipv4Address, 0, 4) + flowStartSeconds = entities.NewInfoElement("flowStartSeconds", 150, entities.DateTimeSeconds, 0, 4) + flowEndSeconds = entities.NewInfoElement("flowEndSeconds", 151, entities.DateTimeSeconds, 0, 4) +) + +// IPFIXClient for IPFIX tests +type IPFIXClient struct { + conn net.Conn +} + +// NewIPFIXClient returns an IPFIXClient that sends data to the given port +func NewIPFIXClient(port int) (*IPFIXClient, error) { + conn, err := net.Dial("udp", fmt.Sprintf(":%d", port)) + if err != nil { + return nil, fmt.Errorf("can't open UDP connection on port %d :%w", + port, err) + } + return &IPFIXClient{ + conn: conn, + }, nil +} + +// SendTemplate must be executed before sending any flow +func (ke *IPFIXClient) SendTemplate() error { + // TODO: add more fields + templateElements := []entities.InfoElementWithValue{ + entities.NewIPAddressInfoElement(sourceIPv4Address, nil), + entities.NewDateTimeSecondsInfoElement(flowStartSeconds, 0), + entities.NewDateTimeSecondsInfoElement(flowEndSeconds, 0), + } + set := entities.NewSet(false) + if err := set.PrepareSet(entities.Template, templateID); err != nil { + return err + } + if err := set.AddRecord(templateElements, templateID); err != nil { + return nil + } + set.UpdateLenInHeader() + return ke.sendMessage(set) +} + +// SendFlow containing the information passed as an argument +func (ke *IPFIXClient) SendFlow(timestamp uint32, srcIP string) error { + // TODO: add more fields + templateElements := []entities.InfoElementWithValue{ + entities.NewIPAddressInfoElement(sourceIPv4Address, net.ParseIP(srcIP)), + entities.NewDateTimeSecondsInfoElement(flowStartSeconds, timestamp), + entities.NewDateTimeSecondsInfoElement(flowEndSeconds, timestamp), + } + set := entities.NewSet(false) + if err := set.PrepareSet(entities.Data, templateID); err != nil { + return err + } + if err := set.AddRecord(templateElements, templateID); err != nil { + return nil + } + set.UpdateLenInHeader() + return ke.sendMessage(set) +} + +func (ke *IPFIXClient) sendMessage(set entities.Set) error { + msg := entities.NewMessage(false) + msg.SetVersion(10) + msg.AddSet(set) + msgLen := entities.MsgHeaderLength + set.GetSetLength() + msg.SetVersion(10) + msg.SetObsDomainID(observationDomainID) + msg.SetMessageLen(uint16(msgLen)) + msg.SetExportTime(uint32(time.Now().Unix())) + msg.SetSequenceNum(0) + bytesSlice := make([]byte, msgLen) + copy(bytesSlice[:entities.MsgHeaderLength], msg.GetMsgHeader()) + copy(bytesSlice[entities.MsgHeaderLength:entities.MsgHeaderLength+entities.SetHeaderLen], set.GetHeaderBuffer()) + index := entities.MsgHeaderLength + entities.SetHeaderLen + for _, record := range set.GetRecords() { + len := record.GetRecordLength() + copy(bytesSlice[index:index+len], record.GetBuffer()) + index += len + } + _, err := ke.conn.Write(bytesSlice) + return err +} + +// UDPPort asks the kernel for a free open port that is ready to use. +func UDPPort() (int, error) { + addr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0") + if err != nil { + return 0, err + } + + l, err := net.ListenUDP("udp", addr) + if err != nil { + return 0, err + } + defer l.Close() + return l.LocalAddr().(*net.UDPAddr).Port, nil +} diff --git a/pkg/test/utils.go b/pkg/test/utils.go index c367d44a1..107fd2a38 100644 --- a/pkg/test/utils.go +++ b/pkg/test/utils.go @@ -19,11 +19,17 @@ package test import ( "bytes" + "encoding/json" "fmt" + "github.com/golang/snappy" jsoniter "github.com/json-iterator/go" "github.com/netobserv/flowlogs-pipeline/pkg/config" + "github.com/netobserv/loki-client-go/pkg/logproto" + log "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stretchr/testify/require" + "io/ioutil" + "net/http" "reflect" "testing" ) @@ -104,3 +110,51 @@ func GetExtractMockEntry() config.GenericMap { } return entry } + +// FakeLokiHandler is a fake loki HTTP service that decodes the snappy/protobuf messages +// and forwards them for later assertions +func FakeLokiHandler(flowsData chan<- map[string]interface{}) http.HandlerFunc { + hlog := log.WithField("component", "LokiHandler") + return func(rw http.ResponseWriter, req *http.Request) { + hlog.WithFields(log.Fields{ + "method": req.Method, + "url": req.URL, + "header": req.Header, + }).Info("new request") + if req.Method != http.MethodPost && req.Method != http.MethodPut { + rw.WriteHeader(http.StatusBadRequest) + return + } + body, err := ioutil.ReadAll(req.Body) + if err != nil { + hlog.WithError(err).Error("can't read request body") + rw.WriteHeader(http.StatusBadRequest) + return + } + decodedBody, err := snappy.Decode([]byte{}, body) + if err != nil { + hlog.WithError(err).Error("can't decode snappy body") + rw.WriteHeader(http.StatusBadRequest) + return + } + pr := logproto.PushRequest{} + if err := pr.Unmarshal(decodedBody); err != nil { + hlog.WithError(err).Error("can't decode protobuf body") + rw.WriteHeader(http.StatusBadRequest) + return + } + for _, stream := range pr.Streams { + for _, entry := range stream.Entries { + flowData := map[string]interface{}{} + if err := json.Unmarshal([]byte(entry.Line), &flowData); err != nil { + hlog.WithError(err).Error("expecting JSON line") + rw.WriteHeader(http.StatusBadRequest) + return + } + // TODO: decorate the flow map with extra metadata from the stream entry + flowsData <- flowData + } + } + rw.WriteHeader(http.StatusOK) + } +} From a8ec50ccb3920e08dc648932bdb5abc7666b8b0e Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Thu, 3 Mar 2022 16:28:52 +0100 Subject: [PATCH 2/5] Address KalmanMeth's comments --- docs/api.md | 9 +++++++++ pkg/api/api.go | 1 + pkg/api/transform_enrich.go | 6 ++++++ pkg/config/config.go | 7 +------ pkg/pipeline/pipeline_netobs_test.go | 2 +- pkg/pipeline/transform/netobserv/enrich.go | 5 +++-- pkg/pipeline/transform/netobserv/enrich_test.go | 4 ++-- 7 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 pkg/api/transform_enrich.go diff --git a/docs/api.md b/docs/api.md index 7c70ee973..e7ceb839c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -42,6 +42,7 @@ Following is the supported API format for the netflow collector: collector: hostName: the hostname to listen on port: the port number to listen on + batchMaxLen: the number of accumulated flows before being forwarded for processing ## Ingest Kafka API Following is the supported API format for the kafka ingest: @@ -119,4 +120,12 @@ Following is the supported API format for specifying metrics aggregations: By: list of fields on which to aggregate Operation: sum, min, max, or avg RecordKey: internal field on which to perform the operation + +## Kubernetes metadata enrichment rules +Following is the supported API format for specifying some network flow enrichment rules from Kubernetes metadata: + +
+ k8s_enrich:
+         kubeConfigPath: path to kubeconfig file (optional)
+         ipFields: names of flow fields that contain an IP address with the prefix to prepend to the original field name (can be empty)
\ No newline at end of file diff --git a/pkg/api/api.go b/pkg/api/api.go index 36290d2b2..e4203d011 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -33,4 +33,5 @@ type API struct { TransformNetwork TransformNetwork `yaml:"network" doc:"## Transform Network API\nFollowing is the supported API format for network transformations:\n"` WriteLoki WriteLoki `yaml:"loki" doc:"## Write Loki API\nFollowing is the supported API format for writing to loki:\n"` ExtractAggregate AggregateDefinition `yaml:"aggregates" doc:"## Aggregate metrics API\nFollowing is the supported API format for specifying metrics aggregations:\n"` + KubeEnrich KubeEnrich `yaml:"k8s_enrich" doc:"## Kubernetes metadata enrichment rules\nFollowing is the supported API format for specifying some network flow enrichment rules from Kubernetes metadata:\n"` } diff --git a/pkg/api/transform_enrich.go b/pkg/api/transform_enrich.go new file mode 100644 index 000000000..a18cefc13 --- /dev/null +++ b/pkg/api/transform_enrich.go @@ -0,0 +1,6 @@ +package api + +type KubeEnrich struct { + KubeConfigPath string `yaml:"kubeConfigPath" doc:"path to kubeconfig file (optional)"` + IPFields map[string]string `yaml:"ipFields" doc:"names of flow fields that contain an IP address with the prefix to prepend to the original field name (can be empty)"` +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 01b9ca5ab..9b064b29e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -83,12 +83,7 @@ type Transform struct { Type string Generic api.TransformGeneric Network api.TransformNetwork - KubeEnrich KubeEnrich `json:"k8s_enrich"` -} - -type KubeEnrich struct { - KubeConfigPath string `json:"kubeConfigPath"` - IPFields map[string]string `json:"ipFields"` + KubeEnrich api.KubeEnrich `json:"k8s_enrich"` } type Extract struct { diff --git a/pkg/pipeline/pipeline_netobs_test.go b/pkg/pipeline/pipeline_netobs_test.go index cbf1e15fc..21991c421 100644 --- a/pkg/pipeline/pipeline_netobs_test.go +++ b/pkg/pipeline/pipeline_netobs_test.go @@ -78,7 +78,7 @@ func NetobsTestPipeline(t *testing.T) *testPipeline { stopInformers := make(chan struct{}) require.NoError(t, informers.Start(stopInformers)) enricher := &netobserv.Enricher{ - Config: config.KubeEnrich{ + Config: api.KubeEnrich{ IPFields: map[string]string{"SrcAddr": ""}, }, Informers: informers, diff --git a/pkg/pipeline/transform/netobserv/enrich.go b/pkg/pipeline/transform/netobserv/enrich.go index 4bf904387..7929b8c00 100644 --- a/pkg/pipeline/transform/netobserv/enrich.go +++ b/pkg/pipeline/transform/netobserv/enrich.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" + "github.com/netobserv/flowlogs-pipeline/pkg/api" "github.com/netobserv/flowlogs-pipeline/pkg/config" kube "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/kubernetes" "github.com/sirupsen/logrus" @@ -22,11 +23,11 @@ var ownerNameFunc = func(owners interface{}, idx int) string { } type Enricher struct { - Config config.KubeEnrich + Config api.KubeEnrich Informers InformersInterface } -func StartEnricher(ctx context.Context, cfg config.KubeEnrich) (*Enricher, error) { +func StartEnricher(ctx context.Context, cfg api.KubeEnrich) (*Enricher, error) { kubeConfig, err := kube.LoadConfig(cfg.KubeConfigPath) if err != nil { return nil, err diff --git a/pkg/pipeline/transform/netobserv/enrich_test.go b/pkg/pipeline/transform/netobserv/enrich_test.go index 0bb853ce4..3fc8cbc16 100644 --- a/pkg/pipeline/transform/netobserv/enrich_test.go +++ b/pkg/pipeline/transform/netobserv/enrich_test.go @@ -3,14 +3,14 @@ package netobserv import ( "testing" - "github.com/netobserv/flowlogs-pipeline/pkg/config" + "github.com/netobserv/flowlogs-pipeline/pkg/api" "github.com/stretchr/testify/assert" ) func setupSimpleEnricher() (*Enricher, *InformersMock) { informers := new(InformersMock) r := Enricher{ - Config: config.KubeEnrich{ + Config: api.KubeEnrich{ IPFields: map[string]string{ "SrcAddr": "Src", "DstAddr": "Dst", From 21698af0b6b19997bbe7a049917959d59df63264 Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Mon, 7 Mar 2022 10:07:08 +0100 Subject: [PATCH 3/5] Addressed review comments --- pkg/api/api.go | 1 - pkg/api/transform_enrich.go | 6 - pkg/config/config.go | 7 +- pkg/pipeline/ingest/ingest_collector.go | 22 +- pkg/pipeline/ingest/ingest_collector_test.go | 48 +++-- pkg/pipeline/pipeline_builder.go | 6 +- pkg/pipeline/pipeline_netobs_test.go | 201 ------------------ .../transform/kubernetes/kubernetes.go | 3 +- pkg/pipeline/transform/netobserv/enrich.go | 132 ------------ .../transform/netobserv/enrich_test.go | 131 ------------ pkg/pipeline/transform/netobserv/meta.go | 183 ---------------- pkg/pipeline/transform/netobserv/mock.go | 85 -------- pkg/pipeline/transform/transform.go | 7 +- pkg/test/eventually.go | 101 --------- pkg/test/eventually_test.go | 58 ----- pkg/test/loki_mock.go | 59 +++++ pkg/test/utils.go | 54 ----- 17 files changed, 108 insertions(+), 996 deletions(-) delete mode 100644 pkg/api/transform_enrich.go delete mode 100644 pkg/pipeline/pipeline_netobs_test.go delete mode 100644 pkg/pipeline/transform/netobserv/enrich.go delete mode 100644 pkg/pipeline/transform/netobserv/enrich_test.go delete mode 100644 pkg/pipeline/transform/netobserv/meta.go delete mode 100644 pkg/pipeline/transform/netobserv/mock.go delete mode 100644 pkg/test/eventually.go delete mode 100644 pkg/test/eventually_test.go create mode 100644 pkg/test/loki_mock.go diff --git a/pkg/api/api.go b/pkg/api/api.go index e4203d011..36290d2b2 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -33,5 +33,4 @@ type API struct { TransformNetwork TransformNetwork `yaml:"network" doc:"## Transform Network API\nFollowing is the supported API format for network transformations:\n"` WriteLoki WriteLoki `yaml:"loki" doc:"## Write Loki API\nFollowing is the supported API format for writing to loki:\n"` ExtractAggregate AggregateDefinition `yaml:"aggregates" doc:"## Aggregate metrics API\nFollowing is the supported API format for specifying metrics aggregations:\n"` - KubeEnrich KubeEnrich `yaml:"k8s_enrich" doc:"## Kubernetes metadata enrichment rules\nFollowing is the supported API format for specifying some network flow enrichment rules from Kubernetes metadata:\n"` } diff --git a/pkg/api/transform_enrich.go b/pkg/api/transform_enrich.go deleted file mode 100644 index a18cefc13..000000000 --- a/pkg/api/transform_enrich.go +++ /dev/null @@ -1,6 +0,0 @@ -package api - -type KubeEnrich struct { - KubeConfigPath string `yaml:"kubeConfigPath" doc:"path to kubeconfig file (optional)"` - IPFields map[string]string `yaml:"ipFields" doc:"names of flow fields that contain an IP address with the prefix to prepend to the original field name (can be empty)"` -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 9b064b29e..f7e148f6b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -80,10 +80,9 @@ type Decode struct { } type Transform struct { - Type string - Generic api.TransformGeneric - Network api.TransformNetwork - KubeEnrich api.KubeEnrich `json:"k8s_enrich"` + Type string + Generic api.TransformGeneric + Network api.TransformNetwork } type Extract struct { diff --git a/pkg/pipeline/ingest/ingest_collector.go b/pkg/pipeline/ingest/ingest_collector.go index 49e95aff6..2dbdac458 100644 --- a/pkg/pipeline/ingest/ingest_collector.go +++ b/pkg/pipeline/ingest/ingest_collector.go @@ -26,7 +26,7 @@ import ( "time" ms "github.com/mitchellh/mapstructure" - "github.com/netobserv/flowlogs-pipeline/pkg/api" + "github.com/netobserv/flowlogs-pipeline/pkg/config" pUtils "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/utils" goflowFormat "github.com/netsampler/goflow2/format" goflowCommonFormat "github.com/netsampler/goflow2/format/common" @@ -171,28 +171,30 @@ func (ingestC *ingestCollector) processLogLines(out chan<- []interface{}) { } // NewIngestCollector create a new ingester -func NewIngestCollector(params api.IngestCollector) (Ingester, error) { - if params.HostName == "" { +func NewIngestCollector(params config.StageParam) (Ingester, error) { + jsonIngestCollector := params.Ingest.Collector + + if jsonIngestCollector.HostName == "" { return nil, fmt.Errorf("ingest hostname not specified") } - if params.Port == 0 { + if jsonIngestCollector.Port == 0 { return nil, fmt.Errorf("ingest port not specified") } - log.Infof("hostname = %s", params.HostName) - log.Infof("port = %d", params.Port) + log.Infof("hostname = %s", jsonIngestCollector.HostName) + log.Infof("port = %d", jsonIngestCollector.Port) ch := make(chan bool, 1) pUtils.RegisterExitChannel(ch) bml := defaultBatchMaxLength - if params.BatchMaxLen != 0 { - bml = params.BatchMaxLen + if jsonIngestCollector.BatchMaxLen != 0 { + bml = jsonIngestCollector.BatchMaxLen } return &ingestCollector{ - hostname: params.HostName, - port: params.Port, + hostname: jsonIngestCollector.HostName, + port: jsonIngestCollector.Port, exitChan: ch, batchFlushTime: defaultBatchFlushTime, batchMaxLength: bml, diff --git a/pkg/pipeline/ingest/ingest_collector_test.go b/pkg/pipeline/ingest/ingest_collector_test.go index 1b0f93130..d8b16dc13 100644 --- a/pkg/pipeline/ingest/ingest_collector_test.go +++ b/pkg/pipeline/ingest/ingest_collector_test.go @@ -22,7 +22,7 @@ func TestIngest(t *testing.T) { exitChan: make(chan bool), } forwarded := make(chan []interface{}) - defer close(forwarded) + //defer close(forwarded) // GIVEN an IPFIX collector Ingester go ic.Ingest(forwarded) @@ -30,25 +30,33 @@ func TestIngest(t *testing.T) { client, err := test.NewIPFIXClient(collectorPort) require.NoError(t, err) - // The IPFIX client might send information before the Ingester is actually listening, - // so we might need to repeat the submission - test.Eventually(t, timeout, func(t require.TestingT) { - // WHEN the service receives an IPFIX message - require.NoError(t, client.SendTemplate()) - require.NoError(t, client.SendFlow(12345678, "1.2.3.4")) - - select { - case received := <-forwarded: - // THEN the bytes are forwarded - require.NotEmpty(t, received) - require.IsType(t, "string", received[0]) - flow := map[string]interface{}{} - require.NoError(t, json.Unmarshal([]byte(received[0].(string)), &flow)) - assert.EqualValues(t, 12345678, flow["TimeFlowStart"]) - assert.EqualValues(t, 12345678, flow["TimeFlowEnd"]) - assert.Equal(t, "1.2.3.4", flow["SrcAddr"]) - case <-time.After(50 * time.Millisecond): + received := waitForFlows(t, client, forwarded) + require.NotEmpty(t, received) + require.IsType(t, "string", received[0]) + flow := map[string]interface{}{} + require.NoError(t, json.Unmarshal([]byte(received[0].(string)), &flow)) + assert.EqualValues(t, 12345678, flow["TimeFlowStart"]) + assert.EqualValues(t, 12345678, flow["TimeFlowEnd"]) + assert.Equal(t, "1.2.3.4", flow["SrcAddr"]) +} + +// The IPFIX client might send information before the Ingester is actually listening, +// so we might need to repeat the submission until the ingest starts forwarding logs +func waitForFlows(t *testing.T, client *test.IPFIXClient, forwarded chan []interface{}) []interface{} { + var start = time.Now() + for { + if client.SendTemplate() == nil && + client.SendFlow(12345678, "1.2.3.4") == nil { + select { + case received := <-forwarded: + return received + default: + // nothing yet received + } + } + if time.Since(start) > timeout { require.Fail(t, "error waiting for ingester to forward received data") } - }) + time.After(50 * time.Millisecond) + } } diff --git a/pkg/pipeline/pipeline_builder.go b/pkg/pipeline/pipeline_builder.go index 38cec228e..8518f9fd9 100644 --- a/pkg/pipeline/pipeline_builder.go +++ b/pkg/pipeline/pipeline_builder.go @@ -1,7 +1,6 @@ package pipeline import ( - "context" "errors" "fmt" @@ -11,7 +10,6 @@ import ( "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/extract" "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/ingest" "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform" - "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/netobserv" "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/write" "github.com/netobserv/gopipes/pkg/node" log "github.com/sirupsen/logrus" @@ -215,7 +213,7 @@ func getIngester(params config.StageParam) (ingest.Ingester, error) { case "file", "file_loop", "file_chunks": ingester, err = ingest.NewIngestFile(params) case "collector": - ingester, err = ingest.NewIngestCollector(params.Ingest.Collector) + ingester, err = ingest.NewIngestCollector(params) case "kafka": ingester, err = ingest.NewIngestKafka(params) default: @@ -266,8 +264,6 @@ func getTransformer(params config.StageParam) (transform.Transformer, error) { transformer, err = transform.NewTransformNetwork(params) case transform.OperationNone: transformer, err = transform.NewTransformNone() - case transform.OperationK8sEnrich: - transformer, err = netobserv.StartEnricher(context.TODO(), params.Transform.KubeEnrich) default: panic(fmt.Sprintf("`transform` type %s not defined; if no encoder needed, specify `none`", params.Transform.Type)) } diff --git a/pkg/pipeline/pipeline_netobs_test.go b/pkg/pipeline/pipeline_netobs_test.go deleted file mode 100644 index 21991c421..000000000 --- a/pkg/pipeline/pipeline_netobs_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package pipeline - -import ( - "fmt" - "net/http/httptest" - "testing" - "time" - - "github.com/netobserv/flowlogs-pipeline/pkg/api" - "github.com/netobserv/flowlogs-pipeline/pkg/config" - "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/decode" - "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/ingest" - "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/netobserv" - "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/write" - "github.com/netobserv/flowlogs-pipeline/pkg/test" - "github.com/prometheus/common/model" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" -) - -// Tests in this file are aimed at validating the Flowlogs Pipeline configuration for its -// usage within the network observability kube enrichment - -const timeout = 5000 * time.Second - -type testPipeline struct { - pipe *Pipeline - client *test.IPFIXClient - lokiFlows chan map[string]interface{} - fakeloki *httptest.Server - stopInformers chan struct{} -} - -func (p *testPipeline) Close() { - close(p.lokiFlows) - close(p.stopInformers) - p.fakeloki.Close() -} - -// NetobsTestPipeline reproduces a basic network observability pipeline and returns it. -// It is not created from the global YAML text configuration because -// we need a programmatic way to instantiate a pipeline and inject there the informers -func NetobsTestPipeline(t *testing.T) *testPipeline { - // STAGE 1: IPFIX ingester - // Find a free port for the IPFIX collector - ipfixPort, err := test.UDPPort() - require.NoError(t, err) - collector, err := ingest.NewIngestCollector(api.IngestCollector{ - HostName: "0.0.0.0", - Port: ipfixPort, - BatchMaxLen: 1, - }) - require.NoError(t, err) - - // STAGE 2: decoding from flow's JSON data to GenericMAP - decoder, err := decode.NewDecodeJson() - require.NoError(t, err) - - // STAGE 3: Kube Enricher getting its data from a mocked informer - informers := netobserv.NewInformers(fake.NewSimpleClientset(&corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "influxdb-v2", - Namespace: "default", - Annotations: map[string]string{ - "anonotation": "true", - }, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - PodIP: "1.2.3.4", - PodIPs: []corev1.PodIP{{IP: "1.2.3.4"}}, - }, - })) - stopInformers := make(chan struct{}) - require.NoError(t, informers.Start(stopInformers)) - enricher := &netobserv.Enricher{ - Config: api.KubeEnrich{ - IPFields: map[string]string{"SrcAddr": ""}, - }, - Informers: informers, - } - - // STAGE 4: Loki exporter writing to a fake loki server that captures all the forwarded data - lokiFlows := make(chan map[string]interface{}, 256) - fakeLoki := httptest.NewServer(test.FakeLokiHandler(lokiFlows)) - log.Debugf("fake loki URL: %s", fakeLoki.URL) - lokiConfig := config.StageParam{ - Name: "loki", - Write: config.Write{Loki: api.WriteLoki{ - URL: fakeLoki.URL, - TenantID: "foo", - BatchWait: "1s", - BatchSize: 1, - Timeout: "1s", - StaticLabels: map[model.LabelName]model.LabelValue{ - "testApp": "network-observability-test", - }, - }}, - } - // NewWriteLoki requires this to not fail during initialization. - // TODO: simplify configuration and make not depending it on global variables - config.Opt.Parameters = fmt.Sprintf( - `[{"name":"write1","write":{"type":"loki","loki":{"url":"%s","batchSize":1,`+ - `"staticLabels":{"testApp":"network-observability-test"}}}}]`, fakeLoki.URL) - config.Parameters = []config.StageParam{lokiConfig} - lokiExporter, err := write.NewWriteLoki(lokiConfig) - require.NoError(t, err) - - // Injecting all the components into a builder object, this way - // we can provide test components - pipe, err := (&builder{ - createdStages: map[string]interface{}{}, - pipelineEntryMap: map[string]*pipelineEntry{ - "ingest": {stageType: StageIngest, Ingester: collector}, - "decode": {stageType: StageDecode, Decoder: decoder}, - "enrich": {stageType: StageTransform, Transformer: enricher}, - "loki": {stageType: StageWrite, Writer: lokiExporter}, - }, - configStages: []config.Stage{ - {Name: "loki", Follows: "enrich"}, - {Name: "enrich", Follows: "decode"}, - {Name: "decode", Follows: "ingest"}, - }, - }).build() - require.NoError(t, err) - - go pipe.Run() - - client, err := test.NewIPFIXClient(ipfixPort) - require.NoError(t, err) - return &testPipeline{ - pipe: pipe, - client: client, - lokiFlows: lokiFlows, - fakeloki: fakeLoki, - stopInformers: stopInformers, - } -} - -func TestNetobservEndToEnd(t *testing.T) { - // GIVEN a network observability pipeline - pipe := NetobsTestPipeline(t) - defer pipe.Close() - - var flow map[string]interface{} - // Since the collector starts in background, it might take some time to accept a - // flow, so we repeat the first submission until it succeeds - test.Eventually(t, timeout, func(t require.TestingT) { - require.NoError(t, pipe.client.SendTemplate()) - - // WHEN a Pod flow is captured - require.NoError(t, pipe.client.SendFlow(12345678, "1.2.3.4")) - - // THEN the flow data is forwarded to Loki - select { - case flow = <-pipe.lokiFlows: - return - case <-time.After(100 * time.Millisecond): - require.Fail(t, "timeout while waiting for flows") - } - }) - - assert.Equal(t, "1.2.3.4", flow["SrcAddr"]) - assert.EqualValues(t, 12345678, flow["TimeFlowStart"]) - assert.EqualValues(t, 12345678, flow["TimeFlowEnd"]) - - // AND it is decorated with the proper Pod metadata - assert.Equal(t, "influxdb-v2", flow["Pod"]) - assert.Equal(t, "influxdb-v2", flow["Workload"]) - assert.Equal(t, "Pod", flow["WorkloadKind"]) - assert.Equal(t, "default", flow["Namespace"]) - - // AND WHEN a Flow is captured from an unknown Kubernetes entity - require.NoError(t, pipe.client.SendFlow(12345699, "4.3.2.1")) - - // Some flows from the previous resubmissions might still be in queue. We wait until - // the last flow is received - test.Eventually(t, timeout, func(t require.TestingT) { - select { - case flow = <-pipe.lokiFlows: - // THEN the flow data is forwarded anyway - assert.Equal(t, "4.3.2.1", flow["SrcAddr"]) - case <-time.After(timeout): - require.Fail(t, "timeout while waiting for flows") - } - }) - - // THEN it is forwarded anyway - assert.EqualValues(t, 12345699, flow["TimeFlowStart"]) - assert.EqualValues(t, 12345699, flow["TimeFlowEnd"]) - - // BUT without any metadata decoration - assert.NotContains(t, flow, "Pod") - assert.NotContains(t, flow, "Workload") - assert.NotContains(t, flow, "WorkloadKind") - assert.NotContains(t, flow, "Namespace") -} diff --git a/pkg/pipeline/transform/kubernetes/kubernetes.go b/pkg/pipeline/transform/kubernetes/kubernetes.go index ff294ba2c..e326641d1 100644 --- a/pkg/pipeline/transform/kubernetes/kubernetes.go +++ b/pkg/pipeline/transform/kubernetes/kubernetes.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "os" + "path" "time" log "github.com/sirupsen/logrus" @@ -235,7 +236,7 @@ func LoadConfig(kubeConfigPath string) (*rest.Config, error) { if err != nil { return nil, fmt.Errorf("can't get user home dir: %w", err) } - kubeConfigPath = homeDir + "/.kube/config" + kubeConfigPath = path.Join(homeDir, ".kube", "config") } config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath) if err == nil { diff --git a/pkg/pipeline/transform/netobserv/enrich.go b/pkg/pipeline/transform/netobserv/enrich.go deleted file mode 100644 index 7929b8c00..000000000 --- a/pkg/pipeline/transform/netobserv/enrich.go +++ /dev/null @@ -1,132 +0,0 @@ -// Package netobserv implements transformations for Network Observability -package netobserv - -import ( - "context" - "fmt" - "strings" - - "github.com/netobserv/flowlogs-pipeline/pkg/api" - "github.com/netobserv/flowlogs-pipeline/pkg/config" - kube "github.com/netobserv/flowlogs-pipeline/pkg/pipeline/transform/kubernetes" - "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" -) - -var elog = logrus.WithField("module", "reader/Enricher") - -var ownerNameFunc = func(owners interface{}, idx int) string { - owner := owners.([]metav1.OwnerReference)[idx] - return owner.Kind + "/" + owner.Name -} - -type Enricher struct { - Config api.KubeEnrich - Informers InformersInterface -} - -func StartEnricher(ctx context.Context, cfg api.KubeEnrich) (*Enricher, error) { - kubeConfig, err := kube.LoadConfig(cfg.KubeConfigPath) - if err != nil { - return nil, err - } - kubeClient, err := kubernetes.NewForConfig(kubeConfig) - if err != nil { - return nil, err - } - informers := NewInformers(kubeClient) - if err := informers.Start(ctx.Done()); err != nil { - return nil, fmt.Errorf("can't start informers: %w", err) - } - informers.WaitForCacheSync(ctx.Done()) - return &Enricher{ - Config: cfg, - Informers: informers, - }, nil -} - -func (e *Enricher) Transform(record config.GenericMap) config.GenericMap { - for ipField, prefixOut := range e.Config.IPFields { - val, ok := record[ipField] - if !ok { - elog.Infof("Field %s not found in record", ipField) - continue - } - ip, ok := val.(string) - if !ok { - elog.Warnf("String expected for field %s value %v", ipField, val) - continue - } - if pod := e.Informers.PodByIP(ip); pod != nil { - e.enrichPod(record, prefixOut, pod) - } else { - // If there is no Pod for such IP, we try searching for a service - e.enrichService(ip, record, prefixOut) - } - } - return record -} - -func (e *Enricher) enrichService(ip string, record map[string]interface{}, prefixOut string) { - if svc := e.Informers.ServiceByIP(ip); svc != nil { - fillWorkloadRecord(record, prefixOut, "Service", svc.Name, svc.Namespace) - } else { - elog.Warnf("Failed to get Service [ip=%v]", ip) - } -} - -func (e *Enricher) enrichPod(record map[string]interface{}, prefixOut string, pod *v1.Pod) { - fillPodRecord(record, prefixOut, pod) - var warnings []string - if len(pod.OwnerReferences) > 0 { - warnings = e.checkTooMany(warnings, "owners", "pod "+pod.Name, pod.OwnerReferences, len(pod.OwnerReferences), ownerNameFunc) - ref := pod.OwnerReferences[0] - if ref.Kind == "ReplicaSet" { - // Search deeper (e.g. Deployment, DeploymentConfig) - if rs := e.Informers.ReplicaSet(pod.Namespace, ref.Name); rs != nil { - if len(rs.OwnerReferences) > 0 { - warnings = e.checkTooMany(warnings, "owners", "replica "+rs.Name, rs.OwnerReferences, len(rs.OwnerReferences), ownerNameFunc) - ref = rs.OwnerReferences[0] - } - } else { - elog.Warnf("Failed to get ReplicaSet [ns=%s,name=%s]", pod.Namespace, ref.Name) - } - } - fillWorkloadRecord(record, prefixOut, ref.Kind, ref.Name, "") - } else { - // Consider a pod without owner as self-owned - fillWorkloadRecord(record, prefixOut, "Pod", pod.Name, "") - } - if len(warnings) > 0 { - record[prefixOut+"Warn"] = strings.Join(warnings, "; ") - } -} - -func (e *Enricher) checkTooMany(warnings []string, kind, ref string, items interface{}, size int, nameFunc func(interface{}, int) string) []string { - if size > 1 { - var names []string - for i := 0; i < size; i++ { - names = append(names, nameFunc(items, i)) - } - elog.Tracef("%d %s found for %s, using first", size, kind, ref) - warn := fmt.Sprintf("Several %s found for %s: %s", kind, ref, strings.Join(names, ",")) - warnings = append(warnings, warn) - } - return warnings -} - -func fillPodRecord(record map[string]interface{}, prefix string, pod *v1.Pod) { - record[prefix+"Pod"] = pod.Name - record[prefix+"Namespace"] = pod.Namespace - record[prefix+"HostIP"] = pod.Status.HostIP -} - -func fillWorkloadRecord(record map[string]interface{}, prefix, kind, name, ns string) { - record[prefix+"Workload"] = name - record[prefix+"WorkloadKind"] = kind - if ns != "" { - record[prefix+"Namespace"] = ns - } -} diff --git a/pkg/pipeline/transform/netobserv/enrich_test.go b/pkg/pipeline/transform/netobserv/enrich_test.go deleted file mode 100644 index 3fc8cbc16..000000000 --- a/pkg/pipeline/transform/netobserv/enrich_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package netobserv - -import ( - "testing" - - "github.com/netobserv/flowlogs-pipeline/pkg/api" - "github.com/stretchr/testify/assert" -) - -func setupSimpleEnricher() (*Enricher, *InformersMock) { - informers := new(InformersMock) - r := Enricher{ - Config: api.KubeEnrich{ - IPFields: map[string]string{ - "SrcAddr": "Src", - "DstAddr": "Dst", - }, - }, - Informers: informers, - } - return &r, informers -} - -func TestEnrichNoMatch(t *testing.T) { - assert := assert.New(t) - r, informers := setupSimpleEnricher() - - informers.MockPod("test-pod1", "test-namespace", "10.0.0.1", "10.0.0.100") - informers.MockNoMatch("10.0.0.2") - - records := map[string]interface{}{ - "SrcAddr": "10.0.0.1", - "DstAddr": "10.0.0.2", - } - - records = r.Transform(records) - assert.Equal(map[string]interface{}{ - "SrcAddr": "10.0.0.1", - "SrcPod": "test-pod1", - "SrcNamespace": "test-namespace", - "SrcHostIP": "10.0.0.100", - "SrcWorkload": "test-pod1", - "SrcWorkloadKind": "Pod", - "DstAddr": "10.0.0.2", - }, records) -} - -func TestEnrichSinglePods(t *testing.T) { - assert := assert.New(t) - r, informers := setupSimpleEnricher() - - informers.MockPod("test-pod1", "test-namespace", "10.0.0.1", "10.0.0.100") - informers.MockPod("test-pod2", "test-namespace", "10.0.0.2", "10.0.0.100") - - records := map[string]interface{}{ - "SrcAddr": "10.0.0.1", - "DstAddr": "10.0.0.2", - } - - records = r.Transform(records) - assert.Equal(map[string]interface{}{ - "SrcAddr": "10.0.0.1", - "SrcPod": "test-pod1", - "SrcNamespace": "test-namespace", - "SrcHostIP": "10.0.0.100", - "SrcWorkload": "test-pod1", - "SrcWorkloadKind": "Pod", - "DstAddr": "10.0.0.2", - "DstPod": "test-pod2", - "DstNamespace": "test-namespace", - "DstHostIP": "10.0.0.100", - "DstWorkload": "test-pod2", - "DstWorkloadKind": "Pod", - }, records) -} - -func TestEnrichDeploymentPods(t *testing.T) { - assert := assert.New(t) - r, informers := setupSimpleEnricher() - - informers.MockPodInDepl("test-pod1", "test-namespace", "10.0.0.1", "10.0.0.100", "test-rs-1", "test-deployment1") - informers.MockPodInDepl("test-pod2", "test-namespace", "10.0.0.2", "10.0.0.100", "test-rs-2", "test-deployment2") - - records := map[string]interface{}{ - "SrcAddr": "10.0.0.1", - "DstAddr": "10.0.0.2", - } - - records = r.Transform(records) - assert.Equal(map[string]interface{}{ - "SrcAddr": "10.0.0.1", - "SrcPod": "test-pod1", - "SrcNamespace": "test-namespace", - "SrcHostIP": "10.0.0.100", - "SrcWorkload": "test-deployment1", - "SrcWorkloadKind": "Deployment", - "DstAddr": "10.0.0.2", - "DstPod": "test-pod2", - "DstNamespace": "test-namespace", - "DstHostIP": "10.0.0.100", - "DstWorkload": "test-deployment2", - "DstWorkloadKind": "Deployment", - }, records) -} - -func TestEnrichPodAndService(t *testing.T) { - assert := assert.New(t) - r, informers := setupSimpleEnricher() - - informers.MockPod("test-pod1", "test-namespace", "10.0.0.1", "10.0.0.100") - informers.MockService("test-service", "test-namespace", "10.0.0.2") - - records := map[string]interface{}{ - "SrcAddr": "10.0.0.1", - "DstAddr": "10.0.0.2", - } - - records = r.Transform(records) - assert.Equal(map[string]interface{}{ - "SrcAddr": "10.0.0.1", - "SrcPod": "test-pod1", - "SrcNamespace": "test-namespace", - "SrcHostIP": "10.0.0.100", - "SrcWorkload": "test-pod1", - "SrcWorkloadKind": "Pod", - "DstAddr": "10.0.0.2", - "DstNamespace": "test-namespace", - "DstWorkload": "test-service", - "DstWorkloadKind": "Service", - }, records) -} diff --git a/pkg/pipeline/transform/netobserv/meta.go b/pkg/pipeline/transform/netobserv/meta.go deleted file mode 100644 index df89d5c4a..000000000 --- a/pkg/pipeline/transform/netobserv/meta.go +++ /dev/null @@ -1,183 +0,0 @@ -package netobserv - -import ( - "fmt" - "io" - "strings" - "time" - - "github.com/sirupsen/logrus" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" -) - -const ( - // NamespaceSeparator used by the K8s informers library to create indices - // that are composed as namespace/name - NamespaceSeparator = "/" - IndexIP = "IP" -) - -var ilog = logrus.WithFields(logrus.Fields{ - "component": fmt.Sprintf("%T", Informers{}), -}) - -type InformersInterface interface { - PodByIP(ip string) *corev1.Pod - ServiceByIP(ip string) *corev1.Service - ReplicaSet(namespace, name string) *appsv1.ReplicaSet -} - -// Informers access for network observability metric decoration -type Informers struct { - InformersInterface - informerFactory informers.SharedInformerFactory - pods cache.SharedIndexInformer - services cache.SharedIndexInformer - replicaSet cache.SharedIndexInformer -} - -func NewInformers(client kubernetes.Interface) *Informers { - // TODO: configure resync time - factory := informers.NewSharedInformerFactory(client, 1*time.Hour) - pods := factory.Core().V1().Pods().Informer() - if err := pods.AddIndexers(map[string]cache.IndexFunc{ - IndexIP: func(obj interface{}) ([]string, error) { - pod := obj.(*corev1.Pod) - ips := make([]string, 0, len(pod.Status.PodIPs)) - for _, ip := range pod.Status.PodIPs { - // ignoring host-networked Pod IPs - if ip.IP != pod.Status.HostIP { - ips = append(ips, ip.IP) - } - } - return ips, nil - }, - }); err != nil { - // this should never happen, as it only returns error if the informer has - // been alrady started - panic(err) - } - services := factory.Core().V1().Services().Informer() - if err := services.AddIndexers(map[string]cache.IndexFunc{ - IndexIP: func(obj interface{}) ([]string, error) { - spec := obj.(*corev1.Service).Spec - if spec.ClusterIP == corev1.ClusterIPNone { - return []string{}, nil - } - return spec.ClusterIPs, nil - }, - }); err != nil { - panic(err) - } - return &Informers{ - informerFactory: factory, - pods: pods, - services: services, - replicaSet: factory.Apps().V1().ReplicaSets().Informer(), - } -} - -func (i *Informers) Start(stopCh <-chan struct{}) error { - i.informerFactory.Start(stopCh) - return nil -} - -func (i *Informers) WaitForCacheSync(stopCh <-chan struct{}) { - i.informerFactory.WaitForCacheSync(stopCh) -} - -func (i *Informers) PodByIP(ip string) *corev1.Pod { - item, err := i.pods.GetIndexer().ByIndex(IndexIP, ip) - if err != nil { - // should never happen as long as we provide the correct index function - // otherwise it's a bug in our code - panic(err) - } - // our provided indexers only return a key, so it's safe to assume 0<=len()<=1 - if len(item) == 0 { - // not found - return nil - } - // since we are excluding host-networked pods, the relation IP:Pod should be 1:1. - if len(item) > 1 { - ilog.WithFields(logrus.Fields{ - "ip": ip, - "results": len(item), - }).Warn("multiple pods for a single IP. Returning the first pod and ignoring the rest") - } - return item[0].(*corev1.Pod) -} - -func (i *Informers) ServiceByIP(ip string) *corev1.Service { - item, err := i.services.GetIndexer().ByIndex(IndexIP, ip) - if err != nil { - // should never happen as long as we provide the correct index function - // otherwise it's a bug in our code - panic(err) - } - if len(item) == 0 { - // not found - return nil - } - // we assume a 1:1 relation between Service and ClusterIP - if len(item) > 1 { - ilog.WithFields(logrus.Fields{ - "ip": ip, - "results": len(item), - }).Warn("multiple services for a single IP. Returning the first service and ignoring the rest") - } - return item[0].(*corev1.Service) -} - -func (i *Informers) ReplicaSet(namespace, name string) *appsv1.ReplicaSet { - item, ok, err := i.replicaSet.GetIndexer().GetByKey(namespace + NamespaceSeparator + name) - if err != nil { - // should never happen. Otherwise it's a bug in our code - panic(err) - } - if !ok { - return nil - } - return item.(*appsv1.ReplicaSet) -} - -func (i *Informers) DebugInfo(out io.Writer) { - fmt.Fprintln(out, "==== Services") - for _, svc := range i.services.GetStore().ListKeys() { - fmt.Fprintln(out, "-", svc) - } - fmt.Fprintln(out, "==== ReplicaSets") - for _, rs := range i.replicaSet.GetStore().ListKeys() { - rskeys := strings.Split(rs, NamespaceSeparator) - rset := i.ReplicaSet(rskeys[0], rskeys[1]) - fmt.Fprintln(out, "-", rs, "replicas:", rset.Status.Replicas) - } - fmt.Fprintln(out, "==== Pods") - for _, pod := range i.pods.GetStore().ListKeys() { - fmt.Fprintln(out, "-", pod) - } - fmt.Fprintln(out, "=== Pods by IP") - for _, ip := range i.pods.GetIndexer().ListIndexFuncValues(IndexIP) { - pod := i.PodByIP(ip) - if pod.Status.PodIP != ip { - panic("ips not equal") - } - fmt.Fprintln(out, "-", ip, ":", pod.Name) - } - fmt.Fprintln(out, "=== Services by IP") - for _, ip := range i.services.GetIndexer().ListIndexFuncValues(IndexIP) { - svc := i.ServiceByIP(ip) - if svc != nil { - if svc.Spec.ClusterIP != ip { - panic("ips not equal") - } - fmt.Fprintln(out, "-", ip, ":", svc.Name) - } else { - fmt.Fprintln(out, "-", ip, "not found in index") - } - } -} diff --git a/pkg/pipeline/transform/netobserv/mock.go b/pkg/pipeline/transform/netobserv/mock.go deleted file mode 100644 index 5ac535fea..000000000 --- a/pkg/pipeline/transform/netobserv/mock.go +++ /dev/null @@ -1,85 +0,0 @@ -package netobserv - -import ( - "github.com/stretchr/testify/mock" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// InformersMock provides an informers' implementation for unit testing -type InformersMock struct { - mock.Mock - InformersInterface -} - -func (o *InformersMock) PodByIP(ip string) *corev1.Pod { - args := o.Called(ip) - return args.Get(0).(*corev1.Pod) -} - -func (o *InformersMock) ServiceByIP(ip string) *corev1.Service { - args := o.Called(ip) - return args.Get(0).(*corev1.Service) -} - -func (o *InformersMock) ReplicaSet(namespace, name string) *appsv1.ReplicaSet { - args := o.Called(namespace, name) - return args.Get(0).(*appsv1.ReplicaSet) -} - -func fakePod(name, ns, host string) *corev1.Pod { - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns, - }, - Status: corev1.PodStatus{ - HostIP: host, - }, - } -} - -func fakeService(name, ns string) *corev1.Service { - return &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns, - }, - } -} - -func (o *InformersMock) MockPod(name, ns, ip, host string) { - o.On("PodByIP", ip).Return(fakePod(name, ns, host)) -} - -func (o *InformersMock) MockPodInDepl(name, ns, ip, host, rs, depl string) { - pod := fakePod(name, ns, host) - pod.OwnerReferences = append(pod.OwnerReferences, metav1.OwnerReference{ - Name: rs, - Kind: "ReplicaSet", - }) - o.On("PodByIP", ip).Return(pod) - o.On("ReplicaSet", ns, rs).Return(&appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: rs, - Namespace: ns, - OwnerReferences: []metav1.OwnerReference{ - { - Name: depl, - Kind: "Deployment", - }, - }, - }, - }) -} - -func (o *InformersMock) MockService(name, ns, ip string) { - o.On("PodByIP", ip).Return((*corev1.Pod)(nil)) - o.On("ServiceByIP", ip).Return(fakeService(name, ns)) -} - -func (o *InformersMock) MockNoMatch(ip string) { - o.On("PodByIP", ip).Return((*corev1.Pod)(nil)) - o.On("ServiceByIP", ip).Return((*corev1.Service)(nil)) -} diff --git a/pkg/pipeline/transform/transform.go b/pkg/pipeline/transform/transform.go index f536aa713..a92725375 100644 --- a/pkg/pipeline/transform/transform.go +++ b/pkg/pipeline/transform/transform.go @@ -50,10 +50,9 @@ type Definition struct { type Definitions []Definition const ( - OperationGeneric = "generic" - OperationNetwork = "network" - OperationK8sEnrich = "k8sEnrich" - OperationNone = "none" + OperationGeneric = "generic" + OperationNetwork = "network" + OperationNone = "none" ) func ExecuteTransform(transformer Transformer, in []config.GenericMap) []config.GenericMap { diff --git a/pkg/test/eventually.go b/pkg/test/eventually.go deleted file mode 100644 index a117553ea..000000000 --- a/pkg/test/eventually.go +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (C) 2022 IBM, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package test - -import ( - "context" - "errors" - "fmt" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -// Eventually retries a test until it eventually succeeds. If the timeout is reached, the test fails -// with the same failure as its last execution. -func Eventually(t *testing.T, timeout time.Duration, testFunc func(_ require.TestingT)) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - success := make(chan interface{}) - errorCh := make(chan error) - failCh := make(chan error) - - go func() { - for ctx.Err() == nil { - result := testResult{failed: false, errorCh: errorCh, failCh: failCh} - // Executing the function to test - testFunc(&result) - // If the function didn't reported failure and didn't reached timeout - if !result.HasFailed() && ctx.Err() == nil { - success <- 1 - break - } - } - }() - - // Wait for success or timeout - var err, fail error - for { - select { - case <-success: - return - case err = <-errorCh: - case fail = <-failCh: - case <-ctx.Done(): - if err != nil { - t.Error(err) - } else if fail != nil { - t.Error(fail) - } else { - t.Error("timeout while waiting for test to complete") - } - return - } - } -} - -// util class for Eventually -type testResult struct { - sync.RWMutex - failed bool - errorCh chan<- error - failCh chan<- error -} - -func (te *testResult) Errorf(format string, args ...interface{}) { - te.Lock() - te.failed = true - te.Unlock() - te.errorCh <- fmt.Errorf(format, args...) -} - -func (te *testResult) FailNow() { - te.Lock() - te.failed = true - te.Unlock() - te.failCh <- errors.New("test failed") -} - -func (te *testResult) HasFailed() bool { - te.RLock() - defer te.RUnlock() - return te.failed -} diff --git a/pkg/test/eventually_test.go b/pkg/test/eventually_test.go deleted file mode 100644 index 6a23ec815..000000000 --- a/pkg/test/eventually_test.go +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2022 IBM, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package test - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestEventually_Error(t *testing.T) { - innerTest := &testing.T{} - Eventually(innerTest, 10*time.Millisecond, func(t require.TestingT) { - require.True(t, false) - }) - assert.True(t, innerTest.Failed()) -} - -func TestEventually_Fail(t *testing.T) { - innerTest := &testing.T{} - Eventually(innerTest, 10*time.Millisecond, func(t require.TestingT) { - t.FailNow() - }) - assert.True(t, innerTest.Failed()) -} - -func TestEventually_Timeout(t *testing.T) { - innerTest := &testing.T{} - Eventually(innerTest, 10*time.Millisecond, func(t require.TestingT) { - time.Sleep(5 * time.Second) - }) - assert.True(t, innerTest.Failed()) -} - -func TestEventually_Success(t *testing.T) { - num := 3 - Eventually(t, 5*time.Second, func(t require.TestingT) { - require.Equal(t, 0, num) - num-- - }) -} diff --git a/pkg/test/loki_mock.go b/pkg/test/loki_mock.go new file mode 100644 index 000000000..ab5c9ccca --- /dev/null +++ b/pkg/test/loki_mock.go @@ -0,0 +1,59 @@ +package test + +import ( + "encoding/json" + "io/ioutil" + "net/http" + + "github.com/golang/snappy" + "github.com/netobserv/loki-client-go/pkg/logproto" + log "github.com/sirupsen/logrus" +) + +// FakeLokiHandler is a fake loki HTTP service that decodes the snappy/protobuf messages +// and forwards them for later assertions +func FakeLokiHandler(flowsData chan<- map[string]interface{}) http.HandlerFunc { + hlog := log.WithField("component", "LokiHandler") + return func(rw http.ResponseWriter, req *http.Request) { + hlog.WithFields(log.Fields{ + "method": req.Method, + "url": req.URL, + "header": req.Header, + }).Info("new request") + if req.Method != http.MethodPost && req.Method != http.MethodPut { + rw.WriteHeader(http.StatusBadRequest) + return + } + body, err := ioutil.ReadAll(req.Body) + if err != nil { + hlog.WithError(err).Error("can't read request body") + rw.WriteHeader(http.StatusBadRequest) + return + } + decodedBody, err := snappy.Decode([]byte{}, body) + if err != nil { + hlog.WithError(err).Error("can't decode snappy body") + rw.WriteHeader(http.StatusBadRequest) + return + } + pr := logproto.PushRequest{} + if err := pr.Unmarshal(decodedBody); err != nil { + hlog.WithError(err).Error("can't decode protobuf body") + rw.WriteHeader(http.StatusBadRequest) + return + } + for _, stream := range pr.Streams { + for _, entry := range stream.Entries { + flowData := map[string]interface{}{} + if err := json.Unmarshal([]byte(entry.Line), &flowData); err != nil { + hlog.WithError(err).Error("expecting JSON line") + rw.WriteHeader(http.StatusBadRequest) + return + } + // TODO: decorate the flow map with extra metadata from the stream entry + flowsData <- flowData + } + } + rw.WriteHeader(http.StatusOK) + } +} diff --git a/pkg/test/utils.go b/pkg/test/utils.go index f624d29a1..e0960b3fd 100644 --- a/pkg/test/utils.go +++ b/pkg/test/utils.go @@ -19,18 +19,12 @@ package test import ( "bytes" - "encoding/json" "fmt" - "io/ioutil" - "net/http" "reflect" "testing" - "github.com/golang/snappy" jsoniter "github.com/json-iterator/go" "github.com/netobserv/flowlogs-pipeline/pkg/config" - "github.com/netobserv/loki-client-go/pkg/logproto" - log "github.com/sirupsen/logrus" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -119,51 +113,3 @@ func GetExtractMockEntry() config.GenericMap { } return entry } - -// FakeLokiHandler is a fake loki HTTP service that decodes the snappy/protobuf messages -// and forwards them for later assertions -func FakeLokiHandler(flowsData chan<- map[string]interface{}) http.HandlerFunc { - hlog := log.WithField("component", "LokiHandler") - return func(rw http.ResponseWriter, req *http.Request) { - hlog.WithFields(log.Fields{ - "method": req.Method, - "url": req.URL, - "header": req.Header, - }).Info("new request") - if req.Method != http.MethodPost && req.Method != http.MethodPut { - rw.WriteHeader(http.StatusBadRequest) - return - } - body, err := ioutil.ReadAll(req.Body) - if err != nil { - hlog.WithError(err).Error("can't read request body") - rw.WriteHeader(http.StatusBadRequest) - return - } - decodedBody, err := snappy.Decode([]byte{}, body) - if err != nil { - hlog.WithError(err).Error("can't decode snappy body") - rw.WriteHeader(http.StatusBadRequest) - return - } - pr := logproto.PushRequest{} - if err := pr.Unmarshal(decodedBody); err != nil { - hlog.WithError(err).Error("can't decode protobuf body") - rw.WriteHeader(http.StatusBadRequest) - return - } - for _, stream := range pr.Streams { - for _, entry := range stream.Entries { - flowData := map[string]interface{}{} - if err := json.Unmarshal([]byte(entry.Line), &flowData); err != nil { - hlog.WithError(err).Error("expecting JSON line") - rw.WriteHeader(http.StatusBadRequest) - return - } - // TODO: decorate the flow map with extra metadata from the stream entry - flowsData <- flowData - } - } - rw.WriteHeader(http.StatusOK) - } -} From 3b4799d252276209439070a9cfccfb047f69fe6f Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Mon, 7 Mar 2022 10:11:34 +0100 Subject: [PATCH 4/5] regenerated api documentation --- docs/api.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/api.md b/docs/api.md index e7ceb839c..372ac9e78 100644 --- a/docs/api.md +++ b/docs/api.md @@ -120,12 +120,4 @@ Following is the supported API format for specifying metrics aggregations: By: list of fields on which to aggregate Operation: sum, min, max, or avg RecordKey: internal field on which to perform the operation - -## Kubernetes metadata enrichment rules -Following is the supported API format for specifying some network flow enrichment rules from Kubernetes metadata: - -
- k8s_enrich:
-         kubeConfigPath: path to kubeconfig file (optional)
-         ipFields: names of flow fields that contain an IP address with the prefix to prepend to the original field name (can be empty)
\ No newline at end of file From 29b7dd130490116b4e5f113ab50753c16b7ff7c9 Mon Sep 17 00:00:00 2001 From: Mario Macias Date: Mon, 7 Mar 2022 10:12:35 +0100 Subject: [PATCH 5/5] upgraded dependencies --- go.mod | 1 - go.sum | 1 - 2 files changed, 2 deletions(-) diff --git a/go.mod b/go.mod index ff4a3f7cf..a41e66cee 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/go-kit/log v0.2.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect diff --git a/go.sum b/go.sum index d50299807..495af2201 100644 --- a/go.sum +++ b/go.sum @@ -217,7 +217,6 @@ github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPO github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=