diff --git a/build/tools/tools.go b/build/tools/tools.go index 48d04fd6c6..f0709bada2 100644 --- a/build/tools/tools.go +++ b/build/tools/tools.go @@ -1,4 +1,5 @@ -//+build tools +//go:build tools +// +build tools package tools @@ -8,8 +9,8 @@ import ( _ "github.com/golang/mock/mockgen" _ "github.com/golangci/golangci-lint/cmd/golangci-lint" _ "github.com/jstemmer/go-junit-report" - _ "mvdan.cc/gofumpt" - _ "google.golang.org/protobuf/cmd/protoc-gen-go" _ "google.golang.org/grpc/cmd/protoc-gen-go-grpc" + _ "google.golang.org/protobuf/cmd/protoc-gen-go" + _ "mvdan.cc/gofumpt" _ "sigs.k8s.io/controller-tools/cmd/controller-gen" ) diff --git a/hack/toolbox/server/go.mod b/hack/toolbox/server/go.mod new file mode 100644 index 0000000000..fad9e5e4ee --- /dev/null +++ b/hack/toolbox/server/go.mod @@ -0,0 +1,3 @@ +module github.com/Azure/azure-container-networking/hack/toolbox/server + +go 1.17 diff --git a/npm/azure-npm.yaml b/npm/azure-npm.yaml index 8d3908ca14..60991f537f 100644 --- a/npm/azure-npm.yaml +++ b/npm/azure-npm.yaml @@ -15,7 +15,7 @@ metadata: addonmanager.kubernetes.io/mode: EnsureExists rules: - apiGroups: - - "" + - "" resources: - pods - nodes @@ -25,7 +25,7 @@ rules: - list - watch - apiGroups: - - networking.k8s.io + - networking.k8s.io resources: - networkpolicies verbs: @@ -34,7 +34,7 @@ rules: - watch --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: ClusterRoleBinding metadata: name: azure-npm-binding namespace: kube-system @@ -66,20 +66,20 @@ spec: labels: k8s-app: azure-npm annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' - azure.npm/scrapeable: '' + scheduler.alpha.kubernetes.io/critical-pod: "" + azure.npm/scrapeable: "" spec: priorityClassName: system-node-critical tolerations: - - operator: "Exists" - effect: NoExecute - - operator: "Exists" - effect: NoSchedule - - key: CriticalAddonsOnly - operator: Exists + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists containers: - name: azure-npm - image: mcr.microsoft.com/containernetworking/azure-npm:v1.4.1 + image: acnpublic.azurecr.io/azure-npm:v1.4.22-24-g0df7a4d7-dirty-v3 resources: limits: cpu: 250m @@ -97,31 +97,31 @@ spec: - name: NPM_CONFIG value: /etc/azure-npm/azure-npm.json volumeMounts: - - name: log - mountPath: /var/log - - name: xtables-lock - mountPath: /run/xtables.lock - - name: protocols - mountPath: /etc/protocols - - name: azure-npm-config - mountPath: /etc/azure-npm + - name: log + mountPath: /var/log + - name: xtables-lock + mountPath: /run/xtables.lock + - name: protocols + mountPath: /etc/protocols + - name: azure-npm-config + mountPath: /etc/azure-npm hostNetwork: true volumes: - - name: log - hostPath: - path: /var/log - type: Directory - - name: xtables-lock - hostPath: - path: /run/xtables.lock - type: File - - name: protocols - hostPath: - path: /etc/protocols - type: File - - name: azure-npm-config - configMap: - name: azure-npm-config + - name: log + hostPath: + path: /var/log + type: Directory + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: File + - name: protocols + hostPath: + path: /etc/protocols + type: File + - name: azure-npm-config + configMap: + name: azure-npm-config serviceAccountName: azure-npm --- apiVersion: v1 @@ -153,7 +153,7 @@ data: "EnablePrometheusMetrics": true, "EnablePprof": true, "EnableHTTPDebugAPI": true, - "EnableV2NPM": false, - "PlaceAzureChainFirst": false + "EnableV2NPM": true, + "PlaceAzureChainFirst": true } } diff --git a/npm/cmd/convertiptable.go b/npm/cmd/convertiptable.go index e7f69e4b08..2d24f729ad 100644 --- a/npm/cmd/convertiptable.go +++ b/npm/cmd/convertiptable.go @@ -21,17 +21,23 @@ func newConvertIPTableCmd() *cobra.Command { c := &dataplane.Converter{} switch { case npmCacheF == "" && iptableSaveF == "": - ipTableRulesRes, err := c.GetJSONRulesFromIptables(iptableName) + ipTableRulesRes, err := c.GetProtobufRulesFromIptable(iptableName) if err != nil { return fmt.Errorf("%w", err) } - fmt.Printf("%s\n", ipTableRulesRes) + + if err := prettyPrintIPTables(ipTableRulesRes); err != nil { + return fmt.Errorf("error printing iptables: %w", err) + } case npmCacheF != "" && iptableSaveF != "": - ipTableRulesRes, err := c.GetJSONRulesFromIptableFile(iptableName, npmCacheF, iptableSaveF) + ipTableRulesRes, err := c.GetProtobufRulesFromIptableFile(iptableName, npmCacheF, iptableSaveF) if err != nil { return fmt.Errorf("%w", err) } - fmt.Printf("%s\n", ipTableRulesRes) + + if err := prettyPrintIPTables(ipTableRulesRes); err != nil { + return fmt.Errorf("error printing iptables from file: %w", err) + } default: return errSpecifyBothFiles } diff --git a/npm/cmd/convertiptable_test.go b/npm/cmd/convertiptable_test.go index 07a6341444..2206164963 100644 --- a/npm/cmd/convertiptable_test.go +++ b/npm/cmd/convertiptable_test.go @@ -44,7 +44,7 @@ func TestConvertIPTableCmd(t *testing.T) { }, { name: "correct files with file order switched", - args: concatArgs(baseArgs, npmCacheFlag, npmCacheFile, iptablesSaveFileFlag, iptableSaveFile), + args: concatArgs(baseArgs, npmCacheFlag, npmCacheFile, iptablesSaveFileFlag, npmCacheFile), wantErr: false, }, { diff --git a/npm/cmd/debug.go b/npm/cmd/debug.go index a23b4fbccc..31d655fd5e 100644 --- a/npm/cmd/debug.go +++ b/npm/cmd/debug.go @@ -3,11 +3,25 @@ package main import ( "fmt" + "github.com/Azure/azure-container-networking/npm/pkg/dataplane/pb" "github.com/spf13/cobra" ) var errSpecifyBothFiles = fmt.Errorf("must specify either no files or both a cache file and an iptables save file") +type IPTablesResponse struct { + Rules map[*pb.RuleResponse]struct{} `json:"rules,omitempty"` +} + +func prettyPrintIPTables(iptableRules map[*pb.RuleResponse]struct{}) error { + iptresponse := IPTablesResponse{ + Rules: iptableRules, + } + + fmt.Printf("%+v", iptresponse) + return nil +} + func newDebugCmd() *cobra.Command { debugCmd := &cobra.Command{ Use: "debug", diff --git a/npm/cmd/debug_test.go b/npm/cmd/debug_test.go index 87ee7c7246..7a025ec2fc 100644 --- a/npm/cmd/debug_test.go +++ b/npm/cmd/debug_test.go @@ -5,12 +5,14 @@ import ( "io" "testing" + dataplane "github.com/Azure/azure-container-networking/npm/pkg/dataplane/debug" + "github.com/Azure/azure-container-networking/npm/util" "github.com/stretchr/testify/require" ) const ( - iptableSaveFile = "../pkg/dataplane/testdata/iptablesave" - npmCacheFile = "../pkg/dataplane/testdata/npmcache.json" + iptableSaveFile = "../pkg/dataplane/testdata/iptablesave-v1" + npmCacheFile = "../pkg/dataplane/testdata/npmcachev1.json" nonExistingFile = "non-existing-iptables-file" npmCacheFlag = "-c" @@ -19,8 +21,8 @@ const ( srcFlag = "-s" unknownShorthandFlag = "-z" - testIP1 = "10.240.0.17" // from npmCacheWithCustomFormat.json - testIP2 = "10.240.0.68" // ditto + testIP1 = "10.224.0.87" // from npmCacheWithCustomFormat.json + testIP2 = "10.224.0.20" // ditto debugCmdString = "debug" convertIPTableCmdString = "convertiptable" @@ -65,3 +67,16 @@ func testCommand(t *testing.T, tests []*testCases) { func concatArgs(baseArgs []string, args ...string) []string { return append(baseArgs, args...) } + +func TestPrettyPrint(t *testing.T) { + c := &dataplane.Converter{} + + iptables, err := c.GetProtobufRulesFromIptableFile( + util.IptablesFilterTable, + npmCacheFile, + iptableSaveFile, + ) + + require.NoError(t, err) + require.NoError(t, prettyPrintIPTables(iptables)) +} diff --git a/npm/cmd/gettuples.go b/npm/cmd/gettuples.go index c4b00229ba..28f9ee9a14 100644 --- a/npm/cmd/gettuples.go +++ b/npm/cmd/gettuples.go @@ -3,9 +3,13 @@ package main import ( "fmt" - dataplane "github.com/Azure/azure-container-networking/npm/pkg/dataplane/debug" + npmconfig "github.com/Azure/azure-container-networking/npm/config" + "github.com/Azure/azure-container-networking/npm/http/api" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" + "github.com/Azure/azure-container-networking/npm/pkg/dataplane/debug" "github.com/Azure/azure-container-networking/npm/util/errors" "github.com/spf13/cobra" + "github.com/spf13/viper" ) func newGetTuples() *cobra.Command { @@ -23,28 +27,46 @@ func newGetTuples() *cobra.Command { } npmCacheF, _ := cmd.Flags().GetString("cache-file") iptableSaveF, _ := cmd.Flags().GetString("iptables-file") - srcType := dataplane.GetInputType(src) - dstType := dataplane.GetInputType(dst) - srcInput := &dataplane.Input{Content: src, Type: srcType} - dstInput := &dataplane.Input{Content: dst, Type: dstType} + srcType := common.GetInputType(src) + dstType := common.GetInputType(dst) + srcInput := &common.Input{Content: src, Type: srcType} + dstInput := &common.Input{Content: dst, Type: dstType} + + config := &npmconfig.Config{} + err := viper.Unmarshal(config) + if err != nil { + return fmt.Errorf("failed to load config with err %w", err) + } switch { case npmCacheF == "" && iptableSaveF == "": - _, tuples, err := dataplane.GetNetworkTuple(srcInput, dstInput) + + c := &debug.Converter{ + NPMDebugEndpointHost: "http://localhost", + NPMDebugEndpointPort: api.DefaultHttpPort, + EnableV2NPM: config.Toggles.EnableV2NPM, // todo: pass this a different way than param to this + } + + _, tuples, srcList, dstList, err := c.GetNetworkTuple(srcInput, dstInput, config) if err != nil { return fmt.Errorf("%w", err) } - for _, tuple := range tuples { - fmt.Printf("%+v\n", tuple) - } + + debug.PrettyPrintTuples(tuples, srcList, dstList) + case npmCacheF != "" && iptableSaveF != "": - _, tuples, err := dataplane.GetNetworkTupleFile(srcInput, dstInput, npmCacheF, iptableSaveF) + + c := &debug.Converter{ + EnableV2NPM: config.Toggles.EnableV2NPM, + } + + _, tuples, srcList, dstList, err := c.GetNetworkTupleFile(srcInput, dstInput, npmCacheF, iptableSaveF) if err != nil { return fmt.Errorf("%w", err) } - for _, tuple := range tuples { - fmt.Printf("%+v\n", tuple) - } + + debug.PrettyPrintTuples(tuples, srcList, dstList) + default: return errSpecifyBothFiles } diff --git a/npm/cmd/parseiptable.go b/npm/cmd/parseiptable.go index 2c1690ee6d..1ef627adfc 100644 --- a/npm/cmd/parseiptable.go +++ b/npm/cmd/parseiptable.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/parse" "github.com/spf13/cobra" ) @@ -14,7 +15,11 @@ func newParseIPTableCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { iptableSaveF, _ := cmd.Flags().GetString("iptables-file") if iptableSaveF == "" { - iptable, err := parse.Iptables("filter") + parser := parse.IPTablesParser{ + IOShim: common.NewIOShim(), + } + + iptable, err := parser.Iptables("filter") if err != nil { return fmt.Errorf("%w", err) } diff --git a/npm/controller/server.go b/npm/controller/server.go index 672b4f3057..5ddc2d0c05 100644 --- a/npm/controller/server.go +++ b/npm/controller/server.go @@ -7,6 +7,7 @@ import ( "fmt" npmconfig "github.com/Azure/azure-container-networking/npm/config" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" controllersv2 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v2" "github.com/Azure/azure-container-networking/npm/pkg/dataplane" "github.com/Azure/azure-container-networking/npm/pkg/models" @@ -87,7 +88,7 @@ func NewNetworkPolicyServer( }, } - n.NpmNamespaceCacheV2 = &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*controllersv2.Namespace)} + n.NpmNamespaceCacheV2 = &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)} n.PodControllerV2 = controllersv2.NewPodController(n.PodInformer, dp, n.NpmNamespaceCacheV2) n.NamespaceControllerV2 = controllersv2.NewNamespaceController(n.NsInformer, dp, n.NpmNamespaceCacheV2) n.NetPolControllerV2 = controllersv2.NewNetworkPolicyController(n.NpInformer, dp) diff --git a/npm/http/server/server.go b/npm/http/server/server.go index 308d038ed7..d20db19103 100644 --- a/npm/http/server/server.go +++ b/npm/http/server/server.go @@ -32,9 +32,8 @@ func NPMRestServerListenAndServe(config npmconfig.Config, npmEncoder json.Marsha rs.router.Handle(api.ClusterMetricsPath, metrics.GetHandler(metrics.ClusterMetrics)) } - // TODO support the debug CLI for v2 // the nil check is for fan-out npm - if config.Toggles.EnableHTTPDebugAPI && npmEncoder != nil && !config.Toggles.EnableV2NPM { + if config.Toggles.EnableHTTPDebugAPI && npmEncoder != nil { // ACN CLI debug handlers rs.router.Handle(api.NPMMgrPath, rs.npmCacheHandler(npmEncoder)).Methods(http.MethodGet) } diff --git a/npm/http/server/server_test.go b/npm/http/server/server_test.go index f23b1b61b2..0cfe2333e6 100644 --- a/npm/http/server/server_test.go +++ b/npm/http/server/server_test.go @@ -9,8 +9,7 @@ import ( "github.com/Azure/azure-container-networking/npm" "github.com/Azure/azure-container-networking/npm/http/api" - "github.com/Azure/azure-container-networking/npm/ipsm" - controllersv1 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v1" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/stretchr/testify/assert" ) @@ -40,18 +39,18 @@ func TestGetNPMCacheHandler(t *testing.T) { t.Errorf("failed to read response's data : %v", err) } - actual := &controllersv1.Cache{} + actual := &common.Cache{} err = json.Unmarshal(byteArray, actual) if err != nil { t.Fatalf("failed to unmarshal %s due to %v", string(byteArray), err) } - expected := &controllersv1.Cache{ + expected := &common.Cache{ NodeName: nodeName, - NsMap: make(map[string]*controllersv1.Namespace), - PodMap: make(map[string]*controllersv1.NpmPod), - ListMap: make(map[string]*ipsm.Ipset), - SetMap: make(map[string]*ipsm.Ipset), + NsMap: make(map[string]*common.Namespace), + PodMap: make(map[string]*common.NpmPod), + ListMap: make(map[string]string), + SetMap: make(map[string]string), } assert.Exactly(expected, actual) diff --git a/npm/ipsm/ipsm.go b/npm/ipsm/ipsm.go index e7f7e8abd4..05074223e0 100644 --- a/npm/ipsm/ipsm.go +++ b/npm/ipsm/ipsm.go @@ -40,7 +40,7 @@ type IpsetManager struct { exec utilexec.Interface listMap map[string]*Ipset // tracks all set lists. setMap map[string]*Ipset // label -> []ip - sync.Mutex + sync.RWMutex } // Ipset represents one ipset entry. @@ -76,28 +76,42 @@ func NewIpsetManager(exec utilexec.Interface) *IpsetManager { } } -func (ipsMgr *IpsetManager) MarshalListMapJSON() ([]byte, error) { - ipsMgr.Lock() - defer ipsMgr.Unlock() +func (ipsMgr *IpsetManager) GetListMapRaw() ([]byte, error) { + ipsMgr.RLock() + defer ipsMgr.RUnlock() + + listMap := make(map[string]string, len(ipsMgr.listMap)) + + for k := range ipsMgr.setMap { + hashedName := util.GetHashedName(k) + listMap[hashedName] = k + } - listMapRaw, err := json.Marshal(ipsMgr.listMap) + b, err := json.Marshal(listMap) if err != nil { - return nil, errors.Errorf("failed to marshal ListMap due to %v", err) + return nil, errors.Wrapf(err, "failed to marshal list map") } - return listMapRaw, nil + return b, nil } -func (ipsMgr *IpsetManager) MarshalSetMapJSON() ([]byte, error) { - ipsMgr.Lock() - defer ipsMgr.Unlock() +func (ipsMgr *IpsetManager) GetSetMapRaw() ([]byte, error) { + ipsMgr.RLock() + defer ipsMgr.RUnlock() + + setMap := make(map[string]string, len(ipsMgr.setMap)) + + for k := range ipsMgr.setMap { + hashedName := util.GetHashedName(k) + setMap[hashedName] = k + } - setMapRaw, err := json.Marshal(ipsMgr.setMap) + b, err := json.Marshal(setMap) if err != nil { - return nil, errors.Errorf("failed to marshal SetMap due to %v", err) + return nil, errors.Wrapf(err, "failed to marshal list map") } - return setMapRaw, nil + return b, nil } // Exists checks if an element exists in setMap/listMap. diff --git a/npm/ipsm/ipsm_test.go b/npm/ipsm/ipsm_test.go index 8d470db51e..7987b518cf 100644 --- a/npm/ipsm/ipsm_test.go +++ b/npm/ipsm/ipsm_test.go @@ -583,11 +583,11 @@ func TestMarshalListMapJSON(t *testing.T) { err := ipsMgr.CreateListNoLock(testListSet) require.NoError(t, err) - listMapRaw, err := ipsMgr.MarshalListMapJSON() + listMapRaw, err := ipsMgr.GetListMapRaw() require.NoError(t, err) fmt.Println(string(listMapRaw)) - expect := []byte(`{"test-list":{}}`) + expect := []byte(`{}`) fmt.Printf("%v\n", ipsMgr.listMap) assert.ElementsMatch(t, expect, listMapRaw) @@ -606,11 +606,11 @@ func TestMarshalSetMapJSON(t *testing.T) { err := ipsMgr.CreateSetNoLock(testSet, []string{util.IpsetNetHashFlag}) require.NoError(t, err) - setMapRaw, err := ipsMgr.MarshalSetMapJSON() + setMapRaw, err := ipsMgr.GetSetMapRaw() require.NoError(t, err) fmt.Println(string(setMapRaw)) - expect := []byte(`{"test-set":{}}`) + expect := []byte(`{"azure-npm-922816856":"test-set"}`) for key, val := range ipsMgr.setMap { fmt.Printf("key: %s value: %+v\n", key, val) } diff --git a/npm/npm.go b/npm/npm.go index bfbaee8dfd..0f8ecfb531 100644 --- a/npm/npm.go +++ b/npm/npm.go @@ -8,6 +8,7 @@ import ( npmconfig "github.com/Azure/azure-container-networking/npm/config" "github.com/Azure/azure-container-networking/npm/ipsm" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" controllersv1 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v1" controllersv2 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v2" "github.com/Azure/azure-container-networking/npm/pkg/dataplane" @@ -26,6 +27,8 @@ var aiMetadata string //nolint // aiMetadata is set in Makefile type NetworkPolicyManager struct { config npmconfig.Config + Dataplane dataplane.GenericDataplane + // ipsMgr are shared in all controllers. Thus, only one ipsMgr is created for simple management // and uses lock to avoid unintentional race condictions in IpsetManager. ipsMgr *ipsm.IpsetManager @@ -55,7 +58,8 @@ func NewNetworkPolicyManager(config npmconfig.Config, klog.Infof("API server version: %+v AI metadata %+v", k8sServerVersion, aiMetadata) npMgr := &NetworkPolicyManager{ - config: config, + config: config, + Dataplane: dp, Informers: models.Informers{ InformerFactory: informerFactory, PodInformer: informerFactory.Core().V1().Pods(), @@ -72,7 +76,7 @@ func NewNetworkPolicyManager(config npmconfig.Config, // create v2 NPM specific components. if npMgr.config.Toggles.EnableV2NPM { - npMgr.NpmNamespaceCacheV2 = &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*controllersv2.Namespace)} + npMgr.NpmNamespaceCacheV2 = &controllersv2.NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)} npMgr.PodControllerV2 = controllersv2.NewPodController(npMgr.PodInformer, dp, npMgr.NpmNamespaceCacheV2) npMgr.NamespaceControllerV2 = controllersv2.NewNamespaceController(npMgr.NsInformer, dp, npMgr.NpmNamespaceCacheV2) // Question(jungukcho): Is config.Toggles.PlaceAzureChainFirst needed for v2? @@ -83,66 +87,77 @@ func NewNetworkPolicyManager(config npmconfig.Config, // create v1 NPM specific components. npMgr.ipsMgr = ipsm.NewIpsetManager(exec) - npMgr.NpmNamespaceCacheV1 = &controllersv1.NpmNamespaceCache{NsMap: make(map[string]*controllersv1.Namespace)} + npMgr.NpmNamespaceCacheV1 = &controllersv1.NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)} npMgr.PodControllerV1 = controllersv1.NewPodController(npMgr.PodInformer, npMgr.ipsMgr, npMgr.NpmNamespaceCacheV1) npMgr.NamespaceControllerV1 = controllersv1.NewNameSpaceController(npMgr.NsInformer, npMgr.ipsMgr, npMgr.NpmNamespaceCacheV1) npMgr.NetPolControllerV1 = controllersv1.NewNetworkPolicyController(npMgr.NpInformer, npMgr.ipsMgr, config.Toggles.PlaceAzureChainFirst) return npMgr } +// Dear Time Traveler: +// This is the server end of the debug dragons den. Several of these properties of the +// npMgr struct have overridden methods which override the MarshalJson, just as this one +// is doing for npMgr. For example, npMgr.NamespaceCacheV2 does not marshal the whole struct, +// but rather the NsMap of type map[string]*Namespace. When unmarshaling, expect this type, +// and pay very close attention. Many hours have been wasted here when unmarshaling mismatched types. func (npMgr *NetworkPolicyManager) MarshalJSON() ([]byte, error) { m := map[models.CacheKey]json.RawMessage{} - var npmNamespaceCacheRaw []byte - var err error if npMgr.config.Toggles.EnableV2NPM { - npmNamespaceCacheRaw, err = json.Marshal(npMgr.NpmNamespaceCacheV2) - } else { - npmNamespaceCacheRaw, err = json.Marshal(npMgr.NpmNamespaceCacheV1) - } + npmNamespaceCacheRaw, err := json.Marshal(npMgr.NpmNamespaceCacheV2) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal v2 ns cache") + } + m[models.NsMap] = npmNamespaceCacheRaw - if err != nil { - return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err) - } - m[models.NsMap] = npmNamespaceCacheRaw + podControllerRaw, err := json.Marshal(npMgr.PodControllerV2) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal v2 pod controller") + } + m[models.PodMap] = podControllerRaw + + setMapRaw, err := json.Marshal(npMgr.Dataplane.GetAllIPSets()) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal v2 set map") + } + m[models.SetMap] = setMapRaw - var podControllerRaw []byte - if npMgr.config.Toggles.EnableV2NPM { - podControllerRaw, err = json.Marshal(npMgr.PodControllerV2) } else { - podControllerRaw, err = json.Marshal(npMgr.PodControllerV1) - } + npmNamespaceCacheRaw, err := json.Marshal(npMgr.NpmNamespaceCacheV1) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal v1 ns cache") + } + m[models.NsMap] = npmNamespaceCacheRaw - if err != nil { - return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err) - } - m[models.PodMap] = podControllerRaw - - // TODO(jungukcho): NPM debug may be broken. - // Will fix it later after v2 controller and linux test if it is broken. - if !npMgr.config.Toggles.EnableV2NPM && npMgr.ipsMgr != nil { - listMapRaw, listMapMarshalErr := npMgr.ipsMgr.MarshalListMapJSON() - if listMapMarshalErr != nil { - return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, listMapMarshalErr) + podControllerRaw, err := json.Marshal(npMgr.PodControllerV1) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal v1 pod controller") + } + m[models.PodMap] = podControllerRaw + + listMapRaw, err := npMgr.ipsMgr.GetListMapRaw() + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal v1 list map") } m[models.ListMap] = listMapRaw - setMapRaw, setMapMarshalErr := npMgr.ipsMgr.MarshalSetMapJSON() - if setMapMarshalErr != nil { - return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, setMapMarshalErr) + setMapRaw, err := npMgr.ipsMgr.GetSetMapRaw() + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal v1 set map") } m[models.SetMap] = setMapRaw + } - nodeNameRaw, err := json.Marshal(npMgr.NodeName) + nodenameRaw, err := json.Marshal(npMgr.NodeName) if err != nil { - return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err) + return nil, errors.Wrapf(err, "failed to marshal node name") } - m[models.NodeName] = nodeNameRaw + m[models.NodeName] = nodenameRaw npmCacheRaw, err := json.Marshal(m) if err != nil { - return nil, errors.Errorf("%s: %v", models.ErrMarshalNPMCache, err) + return nil, errors.Wrapf(err, "failed to marshall the cache map") } return npmCacheRaw, nil diff --git a/npm/npm_test.go b/npm/npm_test.go new file mode 100644 index 0000000000..618c4b87e0 --- /dev/null +++ b/npm/npm_test.go @@ -0,0 +1,66 @@ +package npm + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + npmconfig "github.com/Azure/azure-container-networking/npm/config" + "github.com/Azure/azure-container-networking/npm/http/api" + controllersv2 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v2" + "github.com/Azure/azure-container-networking/npm/pkg/dataplane/debug" + dpmocks "github.com/Azure/azure-container-networking/npm/pkg/dataplane/mocks" + "github.com/Azure/azure-container-networking/npm/pkg/models" + gomock "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +func TestNPMCache(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + dp := dpmocks.NewMockGenericDataplane(ctrl) + dp.EXPECT().GetAllIPSets() + + npmMgr := NetworkPolicyManager{ + config: npmconfig.Config{ + Toggles: npmconfig.Toggles{ + EnableV2NPM: true, + }, + }, + AzureConfig: models.AzureConfig{ + NodeName: "TestNode", + }, + K8SControllersV2: models.K8SControllersV2{ + NamespaceControllerV2: &controllersv2.NamespaceController{}, + }, + Dataplane: dp, + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != api.NPMMgrPath { + t.Errorf("Expected to request '/fixedvalue', got: %s", r.URL.Path) + } + + w.WriteHeader(http.StatusOK) + encoder := &npmMgr + b, err := json.Marshal(encoder) + require.NoError(t, err) + _, err = w.Write(b) + require.NoError(t, err) + })) + defer server.Close() + + host := strings.Split(server.URL[7:], ":") + hostip := host[0] + + c := &debug.Converter{ + NPMDebugEndpointHost: fmt.Sprintf("http://%s", hostip), + NPMDebugEndpointPort: host[1], + EnableV2NPM: true, + } + require.NoError(t, c.InitConverter()) +} diff --git a/npm/npm_v1_test.go b/npm/npm_v1_test.go index 013b1a9df8..db0f207b8b 100644 --- a/npm/npm_v1_test.go +++ b/npm/npm_v1_test.go @@ -10,7 +10,7 @@ import ( "github.com/Azure/azure-container-networking/npm/ipsm" "github.com/Azure/azure-container-networking/npm/iptm" "github.com/Azure/azure-container-networking/npm/metrics" - controllersv1 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v1" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/stretchr/testify/assert" "k8s.io/utils/exec" ) @@ -25,10 +25,11 @@ const ( func TestMarshalJSONForNilValues(t *testing.T) { npMgr := &NetworkPolicyManager{} + npMgr.ipsMgr = ipsm.NewIpsetManager(exec.New()) npmCacheRaw, err := npMgr.MarshalJSON() assert.NoError(t, err) - expect := []byte(`{"NodeName":"","NsMap":null,"PodMap":null}`) + expect := []byte(`{"ListMap":{},"NodeName":"","NsMap":null,"PodMap":null,"SetMap":{}}`) assert.ElementsMatch(t, expect, npmCacheRaw) } @@ -50,17 +51,17 @@ func TestMarshalUnMarshalJSON(t *testing.T) { npmCacheRaw, err := npmCacheEncoder.MarshalJSON() assert.NoError(t, err) - decodedNPMCache := controllersv1.Cache{} + decodedNPMCache := common.Cache{} if err := json.Unmarshal(npmCacheRaw, &decodedNPMCache); err != nil { t.Errorf("failed to decode %s to NPMCache", npmCacheRaw) } - expected := controllersv1.Cache{ - ListMap: make(map[string]*ipsm.Ipset), + expected := common.Cache{ NodeName: nodeName, - NsMap: make(map[string]*controllersv1.Namespace), - PodMap: make(map[string]*controllersv1.NpmPod), - SetMap: make(map[string]*ipsm.Ipset), + NsMap: make(map[string]*common.Namespace), + PodMap: make(map[string]*common.NpmPod), + SetMap: make(map[string]string), + ListMap: make(map[string]string), } if !reflect.DeepEqual(decodedNPMCache, expected) { diff --git a/npm/pkg/controlplane/controllers/common/cache.go b/npm/pkg/controlplane/controllers/common/cache.go new file mode 100644 index 0000000000..ad5cba85bd --- /dev/null +++ b/npm/pkg/controlplane/controllers/common/cache.go @@ -0,0 +1,100 @@ +package common + +import ( + "errors" + "net" + + "github.com/Azure/azure-container-networking/npm/util" +) + +// error type +var ( + ErrSetNotExist = errors.New("set does not exists") + ErrInvalidIPAddress = errors.New("invalid ipaddress, no equivalent pod found") + ErrInvalidInput = errors.New("invalid input") + ErrSetType = errors.New("invalid set type") +) + +type Input struct { + Content string + Type InputType +} + +// InputType indicates allowed typle for source and destination input +type InputType int32 + +// GetInputType returns the type of the input for GetNetworkTuple. +func GetInputType(input string) InputType { + if input == "External" { + return EXTERNAL + } else if ip := net.ParseIP(input); ip != nil { + return IPADDRS + } else { + return NSPODNAME + } +} + +const ( + // IPADDRS indicates the IP Address input type, example: 10.0.0.1 + IPADDRS InputType = 0 + // NSPODNAME indicates the podname input type, example: x/a + NSPODNAME InputType = 1 + // EXTERNAL indicates the external input type + EXTERNAL InputType = 2 +) + +type GenericCache interface { + GetPod(*Input) (*NpmPod, error) + GetNamespaceLabel(namespace string, key string) string + GetListMap() map[string]string + GetSetMap() map[string]string +} + +type Cache struct { + NodeName string + NsMap map[string]*Namespace + PodMap map[string]*NpmPod + SetMap map[string]string + ListMap map[string]string // not used in NPMV2 +} + +func (c *Cache) GetPod(input *Input) (*NpmPod, error) { + switch input.Type { + case NSPODNAME: + if pod, ok := c.PodMap[input.Content]; ok { + return pod, nil + } + return nil, ErrInvalidInput + case IPADDRS: + for _, pod := range c.PodMap { + if pod.PodIP == input.Content { + return pod, nil + } + } + return nil, ErrInvalidIPAddress + case EXTERNAL: + return &NpmPod{}, nil + default: + return nil, ErrInvalidInput + } +} + +func (c *Cache) GetNamespaceLabel(namespace, labelkey string) string { + if _, ok := c.NsMap[namespace]; ok { + return c.NsMap[namespace].LabelsMap[labelkey] + } + return "" +} + +func (c *Cache) GetSetMap() map[string]string { + return c.SetMap +} + +func (c *Cache) GetListMap() map[string]string { + listMap := make(map[string]string) + for k := range c.ListMap { + hashedName := util.GetHashedName(k) + listMap[hashedName] = k + } + return listMap +} diff --git a/npm/pkg/controlplane/controllers/common/namespace.go b/npm/pkg/controlplane/controllers/common/namespace.go new file mode 100644 index 0000000000..ad8a07aa85 --- /dev/null +++ b/npm/pkg/controlplane/controllers/common/namespace.go @@ -0,0 +1,47 @@ +package common + +import ( + "github.com/Azure/azure-container-networking/npm/util" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type Namespace struct { + Name string + LabelsMap map[string]string // Namespace labels +} + +// newNS constructs a new namespace object. +func NewNs(name string) *Namespace { + ns := &Namespace{ + Name: name, + LabelsMap: make(map[string]string), + } + return ns +} + +func (nsObj *Namespace) AppendLabels(newm map[string]string, clear LabelAppendOperation) { + if clear { + nsObj.LabelsMap = make(map[string]string) + } + for k, v := range newm { + nsObj.LabelsMap[k] = v + } +} + +func (nsObj *Namespace) RemoveLabelsWithKey(key string) { + delete(nsObj.LabelsMap, key) +} + +func (nsObj *Namespace) GetNamespaceObjFromNsObj() *corev1.Namespace { + return &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsObj.Name, + Labels: nsObj.LabelsMap, + }, + } +} + +func IsSystemNs(nsObj *corev1.Namespace) bool { + return nsObj.ObjectMeta.Name == util.KubeSystemFlag +} diff --git a/npm/pkg/controlplane/controllers/common/pod.go b/npm/pkg/controlplane/controllers/common/pod.go new file mode 100644 index 0000000000..4a26086d08 --- /dev/null +++ b/npm/pkg/controlplane/controllers/common/pod.go @@ -0,0 +1,91 @@ +package common + +import ( + "reflect" + + corev1 "k8s.io/api/core/v1" + k8slabels "k8s.io/apimachinery/pkg/labels" +) + +type NpmPod struct { + Name string + Namespace string + PodIP string + Labels map[string]string + ContainerPorts []corev1.ContainerPort + Phase corev1.PodPhase +} + +type LabelAppendOperation bool + +const ( + ClearExistingLabels LabelAppendOperation = true + AppendToExistingLabels LabelAppendOperation = false +) + +func (n *NpmPod) IP() string { + return n.PodIP +} + +func (n *NpmPod) NamespaceString() string { + return n.Namespace +} + +func NewNpmPod(podObj *corev1.Pod) *NpmPod { + return &NpmPod{ + Name: podObj.ObjectMeta.Name, + Namespace: podObj.ObjectMeta.Namespace, + PodIP: podObj.Status.PodIP, + Labels: make(map[string]string), + ContainerPorts: []corev1.ContainerPort{}, + Phase: podObj.Status.Phase, + } +} + +func (n *NpmPod) AppendLabels(newPod map[string]string, clear LabelAppendOperation) { + if clear { + n.Labels = make(map[string]string) + } + for k, v := range newPod { + n.Labels[k] = v + } +} + +func (n *NpmPod) RemoveLabelsWithKey(key string) { + delete(n.Labels, key) +} + +func (n *NpmPod) AppendContainerPorts(podObj *corev1.Pod) { + n.ContainerPorts = GetContainerPortList(podObj) +} + +func (n *NpmPod) RemoveContainerPorts() { + n.ContainerPorts = []corev1.ContainerPort{} +} + +// This function can be expanded to other attribs if needed +func (n *NpmPod) UpdateNpmPodAttributes(podObj *corev1.Pod) { + if n.Phase != podObj.Status.Phase { + n.Phase = podObj.Status.Phase + } +} + +// noUpdate evaluates whether NpmPod is required to be update given podObj. +func (n *NpmPod) NoUpdate(podObj *corev1.Pod) bool { + return n.Namespace == podObj.ObjectMeta.Namespace && + n.Name == podObj.ObjectMeta.Name && + n.Phase == podObj.Status.Phase && + n.PodIP == podObj.Status.PodIP && + k8slabels.Equals(n.Labels, podObj.ObjectMeta.Labels) && + // TODO(jungukcho) to avoid using DeepEqual for ContainerPorts, + // it needs a precise sorting. Will optimize it later if needed. + reflect.DeepEqual(n.ContainerPorts, GetContainerPortList(podObj)) +} + +func GetContainerPortList(podObj *corev1.Pod) []corev1.ContainerPort { + portList := []corev1.ContainerPort{} + for _, container := range podObj.Spec.Containers { //nolint:gocritic // intentionally copying full struct :( + portList = append(portList, container.Ports...) //nolint:gocritic // intentionally copying full struct :( + } + return portList +} diff --git a/npm/pkg/controlplane/controllers/v1/nameSpaceController.go b/npm/pkg/controlplane/controllers/v1/nameSpaceController.go index 0809ab42a5..3fa1b55de2 100644 --- a/npm/pkg/controlplane/controllers/v1/nameSpaceController.go +++ b/npm/pkg/controlplane/controllers/v1/nameSpaceController.go @@ -10,10 +10,10 @@ import ( "github.com/Azure/azure-container-networking/npm/ipsm" "github.com/Azure/azure-container-networking/npm/metrics" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/Azure/azure-container-networking/npm/util" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8slabels "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -24,19 +24,18 @@ import ( "k8s.io/klog" ) -type LabelAppendOperation bool - -const ( - ClearExistingLabels LabelAppendOperation = true - AppendToExistingLabels LabelAppendOperation = false -) - // NpmNamespaceCache to store namespace struct in nameSpaceController.go. // Since this cache is shared between podController and NameSpaceController, // it has mutex for avoiding racing condition between them. type NpmNamespaceCache struct { - sync.Mutex - NsMap map[string]*Namespace // Key is ns- + sync.RWMutex + NsMap map[string]*common.Namespace // Key is ns- +} + +func (n *NpmNamespaceCache) GetNsMap() map[string]*common.Namespace { + n.RLock() + defer n.RUnlock() + return n.NsMap } func (n *NpmNamespaceCache) MarshalJSON() ([]byte, error) { @@ -51,46 +50,6 @@ func (n *NpmNamespaceCache) MarshalJSON() ([]byte, error) { return nsMapRaw, nil } -type Namespace struct { - name string - LabelsMap map[string]string // NameSpace labels -} - -// newNS constructs a new namespace object. -func newNs(name string) *Namespace { - ns := &Namespace{ - name: name, - LabelsMap: make(map[string]string), - } - return ns -} - -func (nsObj *Namespace) getNamespaceObjFromNsObj() *corev1.Namespace { - return &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: nsObj.name, - Labels: nsObj.LabelsMap, - }, - } -} - -func (nsObj *Namespace) appendLabels(new map[string]string, clear LabelAppendOperation) { - if clear { - nsObj.LabelsMap = make(map[string]string) - } - for k, v := range new { - nsObj.LabelsMap[k] = v - } -} - -func (nsObj *Namespace) removeLabelsWithKey(key string) { - delete(nsObj.LabelsMap, key) -} - -func isSystemNs(nsObj *corev1.Namespace) bool { - return nsObj.ObjectMeta.Name == util.KubeSystemFlag -} - type NamespaceController struct { nameSpaceLister corelisters.NamespaceLister workqueue workqueue.RateLimitingInterface @@ -343,7 +302,7 @@ func (nsc *NamespaceController) syncAddNameSpace(nsObj *corev1.Namespace) error return err } - npmNs := newNs(corev1NsName) + npmNs := common.NewNs(corev1NsName) nsc.npmNamespaceCache.NsMap[corev1NsName] = npmNs // Add the namespace to its label's ipset list. @@ -363,7 +322,7 @@ func (nsc *NamespaceController) syncAddNameSpace(nsObj *corev1.Namespace) error } // Append succeeded labels to the cache NS obj - npmNs.appendLabels(map[string]string{nsLabelKey: nsLabelVal}, AppendToExistingLabels) + npmNs.AppendLabels(map[string]string{nsLabelKey: nsLabelVal}, common.AppendToExistingLabels) } return nil @@ -405,7 +364,7 @@ func (nsc *NamespaceController) syncUpdateNameSpace(newNsObj *corev1.Namespace) // (TODO) need to remove this ordering dependency removedLabelKey, removedLabelValue := util.GetLabelKVFromSet(nsLabelVal) if removedLabelValue != "" { - curNsObj.removeLabelsWithKey(removedLabelKey) + curNsObj.RemoveLabelsWithKey(removedLabelKey) } } @@ -421,14 +380,14 @@ func (nsc *NamespaceController) syncUpdateNameSpace(newNsObj *corev1.Namespace) // only after both ipsets for a given label's key value pair are added successfully addedLabelKey, addedLabelValue := util.GetLabelKVFromSet(nsLabelVal) if addedLabelValue != "" { - curNsObj.appendLabels(map[string]string{addedLabelKey: addedLabelValue}, AppendToExistingLabels) + curNsObj.AppendLabels(map[string]string{addedLabelKey: addedLabelValue}, common.AppendToExistingLabels) } } // Append all labels to the cache NS obj // If due to ordering issue the above deleted and added labels are not correct, // this below appendLabels will help ensure correct state in cache for all successful ops. - curNsObj.appendLabels(newNsLabel, ClearExistingLabels) + curNsObj.AppendLabels(newNsLabel, common.ClearExistingLabels) nsc.npmNamespaceCache.NsMap[newNsName] = curNsObj return metrics.UpdateOp, nil @@ -462,7 +421,7 @@ func (nsc *NamespaceController) cleanDeletedNamespace(cachedNsKey string) error } // remove labels from the cache NS obj - cachedNsObj.removeLabelsWithKey(nsLabelKey) + cachedNsObj.RemoveLabelsWithKey(nsLabelKey) } // Delete the namespace from all-namespace ipset list. diff --git a/npm/pkg/controlplane/controllers/v1/nameSpaceController_test.go b/npm/pkg/controlplane/controllers/v1/nameSpaceController_test.go index 5310969b83..5e95b75afd 100644 --- a/npm/pkg/controlplane/controllers/v1/nameSpaceController_test.go +++ b/npm/pkg/controlplane/controllers/v1/nameSpaceController_test.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/npm/ipsm" "github.com/Azure/azure-container-networking/npm/metrics" "github.com/Azure/azure-container-networking/npm/metrics/promutil" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/Azure/azure-container-networking/npm/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -106,7 +107,7 @@ func (f *nameSpaceFixture) newNsController(stopCh chan struct{}) { kubeclient := k8sfake.NewSimpleClientset(f.kubeobjects...) f.kubeInformer = kubeinformers.NewSharedInformerFactory(kubeclient, noResyncPeriodFunc()) - npmNamespaceCache := &NpmNamespaceCache{NsMap: make(map[string]*Namespace)} + npmNamespaceCache := &NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)} f.nsController = NewNameSpaceController( f.kubeInformer.Core().V1().Namespaces(), f.ipsMgr, npmNamespaceCache) @@ -528,12 +529,12 @@ func TestDeleteNamespaceWithTombstoneAfterAddingNameSpace(t *testing.T) { } func TestGetNamespaceObjFromNsObj(t *testing.T) { - ns := newNs("test-ns") + ns := common.NewNs("test-ns") ns.LabelsMap = map[string]string{ "test": "new", } - nsObj := ns.getNamespaceObjFromNsObj() + nsObj := ns.GetNamespaceObjFromNsObj() if !reflect.DeepEqual(ns.LabelsMap, nsObj.ObjectMeta.Labels) { t.Errorf("TestGetNamespaceObjFromNsObj failed @ nsObj labels check") @@ -543,7 +544,7 @@ func TestGetNamespaceObjFromNsObj(t *testing.T) { func TestIsSystemNs(t *testing.T) { nsObj := newNameSpace("kube-system", "0", map[string]string{"test": "new"}) - if !isSystemNs(nsObj) { + if !common.IsSystemNs(nsObj) { t.Errorf("TestIsSystemNs failed @ nsObj isSystemNs check") } } @@ -562,10 +563,10 @@ func checkNsTestResult(testName string, f *nameSpaceFixture, testCases []expecte } func TestNSMapMarshalJSON(t *testing.T) { - npmNSCache := &NpmNamespaceCache{NsMap: make(map[string]*Namespace)} + npmNSCache := &NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)} nsName := "ns-test" - ns := &Namespace{ - name: nsName, + ns := &common.Namespace{ + Name: nsName, LabelsMap: map[string]string{ "test-key": "test-value", }, @@ -575,6 +576,6 @@ func TestNSMapMarshalJSON(t *testing.T) { nsMapRaw, err := npmNSCache.MarshalJSON() require.NoError(t, err) - expect := []byte(`{"ns-test":{"LabelsMap":{"test-key":"test-value"}}}`) + expect := []byte(`{"ns-test":{"Name":"ns-test","LabelsMap":{"test-key":"test-value"}}}`) assert.ElementsMatch(t, expect, nsMapRaw) } diff --git a/npm/pkg/controlplane/controllers/v1/npmCache.go b/npm/pkg/controlplane/controllers/v1/npmCache.go index 9b9ffe5d93..3a599aa069 100644 --- a/npm/pkg/controlplane/controllers/v1/npmCache.go +++ b/npm/pkg/controlplane/controllers/v1/npmCache.go @@ -1,15 +1,3 @@ // Copyright 2018 Microsoft. All rights reserved. // MIT License package controllers - -import ( - "github.com/Azure/azure-container-networking/npm/ipsm" -) - -type Cache struct { - NodeName string - NsMap map[string]*Namespace - PodMap map[string]*NpmPod - ListMap map[string]*ipsm.Ipset - SetMap map[string]*ipsm.Ipset -} diff --git a/npm/pkg/controlplane/controllers/v1/podController.go b/npm/pkg/controlplane/controllers/v1/podController.go index 30f7ac6a9c..1d38baa683 100644 --- a/npm/pkg/controlplane/controllers/v1/podController.go +++ b/npm/pkg/controlplane/controllers/v1/podController.go @@ -11,8 +11,8 @@ import ( "github.com/Azure/azure-container-networking/npm/ipsm" "github.com/Azure/azure-container-networking/npm/metrics" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/Azure/azure-container-networking/npm/util" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -21,7 +21,6 @@ import ( coreinformer "k8s.io/client-go/informers/core/v1" corelisters "k8s.io/client-go/listers/core/v1" - k8slabels "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" "k8s.io/klog" @@ -35,72 +34,12 @@ const ( addNamedPort NamedPortOperation = "add" ) -type NpmPod struct { - Name string - Namespace string - PodIP string - Labels map[string]string - ContainerPorts []corev1.ContainerPort - Phase corev1.PodPhase -} - -func newNpmPod(podObj *corev1.Pod) *NpmPod { - return &NpmPod{ - Name: podObj.ObjectMeta.Name, - Namespace: podObj.ObjectMeta.Namespace, - PodIP: podObj.Status.PodIP, - Labels: make(map[string]string), - ContainerPorts: []corev1.ContainerPort{}, - Phase: podObj.Status.Phase, - } -} - -func (nPod *NpmPod) appendLabels(new map[string]string, clear LabelAppendOperation) { - if clear { - nPod.Labels = make(map[string]string) - } - for k, v := range new { - nPod.Labels[k] = v - } -} - -func (nPod *NpmPod) removeLabelsWithKey(key string) { - delete(nPod.Labels, key) -} - -func (nPod *NpmPod) appendContainerPorts(podObj *corev1.Pod) { - nPod.ContainerPorts = getContainerPortList(podObj) -} - -func (nPod *NpmPod) removeContainerPorts() { - nPod.ContainerPorts = []corev1.ContainerPort{} -} - -// This function can be expanded to other attribs if needed -func (nPod *NpmPod) updateNpmPodAttributes(podObj *corev1.Pod) { - if nPod.Phase != podObj.Status.Phase { - nPod.Phase = podObj.Status.Phase - } -} - -// noUpdate evaluates whether NpmPod is required to be update given podObj. -func (nPod *NpmPod) noUpdate(podObj *corev1.Pod) bool { - return nPod.Namespace == podObj.ObjectMeta.Namespace && - nPod.Name == podObj.ObjectMeta.Name && - nPod.Phase == podObj.Status.Phase && - nPod.PodIP == podObj.Status.PodIP && - k8slabels.Equals(nPod.Labels, podObj.ObjectMeta.Labels) && - // TODO(jungukcho) to avoid using DeepEqual for ContainerPorts, - // it needs a precise sorting. Will optimize it later if needed. - reflect.DeepEqual(nPod.ContainerPorts, getContainerPortList(podObj)) -} - type PodController struct { podLister corelisters.PodLister workqueue workqueue.RateLimitingInterface ipsMgr *ipsm.IpsetManager - podMap map[string]*NpmPod // Key is / - sync.Mutex + podMap map[string]*common.NpmPod // Key is / + sync.RWMutex npmNamespaceCache *NpmNamespaceCache } @@ -109,7 +48,7 @@ func NewPodController(podInformer coreinformer.PodInformer, ipsMgr *ipsm.IpsetMa podLister: podInformer.Lister(), workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Pods"), ipsMgr: ipsMgr, - podMap: make(map[string]*NpmPod), + podMap: make(map[string]*common.NpmPod), npmNamespaceCache: npmNamespaceCache, } @@ -129,7 +68,7 @@ func (c *PodController) MarshalJSON() ([]byte, error) { podMapRaw, err := json.Marshal(c.podMap) if err != nil { - return nil, errors.Errorf("failed to marshal podMap due to %v", err) + return nil, errors.Wrapf(err, "failed to marshal podMap") } return podMapRaw, nil @@ -372,7 +311,7 @@ func (c *PodController) syncPod(key string) error { // if pod does not have different states against lastly applied states stored in cachedNpmPod, // podController does not need to reconcile this update. // in this updatePod event, newPod was updated with states which PodController does not need to reconcile. - if cachedNpmPod.noUpdate(pod) { + if cachedNpmPod.NoUpdate(pod) { return nil } } @@ -399,7 +338,7 @@ func (c *PodController) syncAddedPod(podObj *corev1.Pod) error { } // Create npmPod and add it to the podMap - npmPodObj := newNpmPod(podObj) + npmPodObj := common.NewNpmPod(podObj) c.podMap[podKey] = npmPodObj // Get lists of podLabelKey and podLabelKey + podLavelValue ,and then start adding them to ipsets. @@ -414,16 +353,16 @@ func (c *PodController) syncAddedPod(podObj *corev1.Pod) error { if err = c.ipsMgr.AddToSet(podIPSetName, npmPodObj.PodIP, util.IpsetNetHashFlag, podKey); err != nil { return fmt.Errorf("[syncAddedPod] Error: failed to add pod to label ipset with err: %w", err) } - npmPodObj.appendLabels(map[string]string{labelKey: labelVal}, AppendToExistingLabels) + npmPodObj.AppendLabels(map[string]string{labelKey: labelVal}, common.AppendToExistingLabels) } // Add pod's named ports from its ipset. klog.Infof("Adding named port ipsets") - containerPorts := getContainerPortList(podObj) + containerPorts := common.GetContainerPortList(podObj) if err = c.manageNamedPortIpsets(containerPorts, podKey, npmPodObj.PodIP, addNamedPort); err != nil { return fmt.Errorf("[syncAddedPod] Error: failed to add pod to named port ipset with err: %w", err) } - npmPodObj.appendContainerPorts(podObj) + npmPodObj.AppendContainerPorts(podObj) return nil } @@ -450,7 +389,7 @@ func (c *PodController) syncAddAndUpdatePod(newPodObj *corev1.Pod) (metrics.Oper } // Add namespace object into NsMap cache only when two ipset operations are successful. - npmNs := newNs(newPodObjNs) + npmNs := common.NewNs(newPodObjNs) c.npmNamespaceCache.NsMap[newPodObjNs] = npmNs } c.npmNamespaceCache.Unlock() @@ -507,7 +446,7 @@ func (c *PodController) syncAddAndUpdatePod(newPodObj *corev1.Pod) (metrics.Oper // key + val ipsets are worked on. 0th index will be key and 1st index will be value of the label removedLabelKey, removedLabelValue := util.GetLabelKVFromSet(podIPSetName) if removedLabelValue != "" { - cachedNpmPod.removeLabelsWithKey(removedLabelKey) + cachedNpmPod.RemoveLabelsWithKey(removedLabelKey) } } @@ -522,18 +461,18 @@ func (c *PodController) syncAddAndUpdatePod(newPodObj *corev1.Pod) (metrics.Oper // (TODO) will need to remove this ordering dependency addedLabelKey, addedLabelValue := util.GetLabelKVFromSet(addIPSetName) if addedLabelValue != "" { - cachedNpmPod.appendLabels(map[string]string{addedLabelKey: addedLabelValue}, AppendToExistingLabels) + cachedNpmPod.AppendLabels(map[string]string{addedLabelKey: addedLabelValue}, common.AppendToExistingLabels) } } // This will ensure after all labels are worked on to overwrite. This way will reduce any bugs introduced above // If due to ordering issue the above deleted and added labels are not correct, // this below appendLabels will help ensure correct state in cache for all successful ops. - cachedNpmPod.appendLabels(newPodObj.Labels, ClearExistingLabels) + cachedNpmPod.AppendLabels(newPodObj.Labels, common.ClearExistingLabels) // (TODO): optimize named port addition and deletions. // named ports are mostly static once configured in todays usage pattern // so keeping this simple by deleting all and re-adding - newPodPorts := getContainerPortList(newPodObj) + newPodPorts := common.GetContainerPortList(newPodObj) if !reflect.DeepEqual(cachedNpmPod.ContainerPorts, newPodPorts) { // Delete cached pod's named ports from its ipset. if err = c.manageNamedPortIpsets( @@ -541,15 +480,15 @@ func (c *PodController) syncAddAndUpdatePod(newPodObj *corev1.Pod) (metrics.Oper return metrics.UpdateOp, fmt.Errorf("[syncAddAndUpdatePod] Error: failed to delete pod from named port ipset with err: %w", err) } // Since portList ipset deletion is successful, NPM can remove cachedContainerPorts - cachedNpmPod.removeContainerPorts() + cachedNpmPod.RemoveContainerPorts() // Add new pod's named ports from its ipset. if err = c.manageNamedPortIpsets(newPodPorts, podKey, newPodObj.Status.PodIP, addNamedPort); err != nil { return metrics.UpdateOp, fmt.Errorf("[syncAddAndUpdatePod] Error: failed to add pod to named port ipset with err: %w", err) } - cachedNpmPod.appendContainerPorts(newPodObj) + cachedNpmPod.AppendContainerPorts(newPodObj) } - cachedNpmPod.updateNpmPodAttributes(newPodObj) + cachedNpmPod.UpdateNpmPodAttributes(newPodObj) return metrics.UpdateOp, nil } @@ -582,7 +521,7 @@ func (c *PodController) cleanUpDeletedPod(cachedNpmPodKey string) error { if err = c.ipsMgr.DeleteFromSet(podIPSetName, cachedNpmPod.PodIP, cachedNpmPodKey); err != nil { return fmt.Errorf("[cleanUpDeletedPod] Error: failed to delete pod from label ipset with err: %w", err) } - cachedNpmPod.removeLabelsWithKey(labelKey) + cachedNpmPod.RemoveLabelsWithKey(labelKey) } // Delete pod's named ports from its ipset. Need to pass true in the manageNamedPortIpsets function call @@ -655,11 +594,3 @@ func hasValidPodIP(podObj *corev1.Pod) bool { func isHostNetworkPod(podObj *corev1.Pod) bool { return podObj.Spec.HostNetwork } - -func getContainerPortList(podObj *corev1.Pod) []corev1.ContainerPort { - portList := []corev1.ContainerPort{} - for _, container := range podObj.Spec.Containers { - portList = append(portList, container.Ports...) - } - return portList -} diff --git a/npm/pkg/controlplane/controllers/v1/podController_test.go b/npm/pkg/controlplane/controllers/v1/podController_test.go index ab10dafe7d..40bc7a88bc 100644 --- a/npm/pkg/controlplane/controllers/v1/podController_test.go +++ b/npm/pkg/controlplane/controllers/v1/podController_test.go @@ -3,6 +3,7 @@ package controllers import ( + "encoding/json" "fmt" "reflect" "strconv" @@ -11,6 +12,7 @@ import ( "github.com/Azure/azure-container-networking/npm/ipsm" "github.com/Azure/azure-container-networking/npm/metrics" "github.com/Azure/azure-container-networking/npm/metrics/promutil" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/Azure/azure-container-networking/npm/util" testutils "github.com/Azure/azure-container-networking/test/utils" "github.com/stretchr/testify/assert" @@ -77,7 +79,7 @@ func (f *podFixture) newPodController(stopCh chan struct{}) { kubeclient := k8sfake.NewSimpleClientset(f.kubeobjects...) f.kubeInformer = kubeinformers.NewSharedInformerFactory(kubeclient, noResyncPeriodFunc()) - npmNamespaceCache := &NpmNamespaceCache{NsMap: make(map[string]*Namespace)} + npmNamespaceCache := &NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)} f.podController = NewPodController(f.kubeInformer.Core().V1().Pods(), f.ipsMgr, npmNamespaceCache) for _, pod := range f.podLister { @@ -241,7 +243,7 @@ func checkNpmPodWithInput(testName string, f *podFixture, inputPodObj *corev1.Po f.t.Errorf("%s failed @ Labels check got = %v, want %v", testName, cachedNpmPodObj.Labels, inputPodObj.Labels) } - inputPortList := getContainerPortList(inputPodObj) + inputPortList := common.GetContainerPortList(inputPodObj) if !reflect.DeepEqual(cachedNpmPodObj.ContainerPorts, inputPortList) { f.t.Errorf("%s failed @ Container port check got = %v, want %v", testName, cachedNpmPodObj.PodIP, inputPortList) } @@ -713,10 +715,10 @@ func TestPodMapMarshalJSON(t *testing.T) { podKey, err := cache.MetaNamespaceKeyFunc(pod) assert.NoError(t, err) - npmPod := newNpmPod(pod) + npmPod := common.NewNpmPod(pod) f.podController.podMap[podKey] = npmPod - npMapRaw, err := f.podController.MarshalJSON() + npMapRaw, err := json.Marshal(f.podController) assert.NoError(t, err) expect := []byte(`{"test-namespace/test-pod":{"Name":"test-pod","Namespace":"test-namespace","PodIP":"1.2.3.4","Labels":{},"ContainerPorts":[],"Phase":"Running"}}`) @@ -906,13 +908,13 @@ func TestNPMPodNoUpdate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() corev1Pod := createPod(tt.podName, tt.ns, tt.rv, tt.podIP, tt.labels, tt.isHostNetwork, tt.podPhase) - npmPod := newNpmPod(corev1Pod) + npmPod := common.NewNpmPod(corev1Pod) if tt.updatingNPMPod { - npmPod.appendLabels(corev1Pod.Labels, AppendToExistingLabels) - npmPod.updateNpmPodAttributes(corev1Pod) - npmPod.appendContainerPorts(corev1Pod) + npmPod.AppendLabels(corev1Pod.Labels, common.AppendToExistingLabels) + npmPod.UpdateNpmPodAttributes(corev1Pod) + npmPod.AppendContainerPorts(corev1Pod) } - noUpdate := npmPod.noUpdate(corev1Pod) + noUpdate := npmPod.NoUpdate(corev1Pod) require.Equal(t, tt.expectedNoUpdate, noUpdate) }) } diff --git a/npm/pkg/controlplane/controllers/v2/namespaceController.go b/npm/pkg/controlplane/controllers/v2/namespaceController.go index 4bed108743..98520c037b 100644 --- a/npm/pkg/controlplane/controllers/v2/namespaceController.go +++ b/npm/pkg/controlplane/controllers/v2/namespaceController.go @@ -10,6 +10,7 @@ import ( "time" "github.com/Azure/azure-container-networking/npm/metrics" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/Azure/azure-container-networking/npm/pkg/dataplane" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets" "github.com/Azure/azure-container-networking/npm/util" @@ -38,13 +39,19 @@ var errWorkqueueFormatting = errors.New("error in formatting") // Since this cache is shared between podController and NamespaceController, // it has mutex for avoiding racing condition between them. type NpmNamespaceCache struct { - sync.Mutex - NsMap map[string]*Namespace // Key is ns- + sync.RWMutex + NsMap map[string]*common.Namespace // Key is ns- +} + +func (c *NpmNamespaceCache) GetCache() map[string]*common.Namespace { + c.RLock() + defer c.RUnlock() + return c.NsMap } func (n *NpmNamespaceCache) MarshalJSON() ([]byte, error) { - n.Lock() - defer n.Unlock() + n.RLock() + defer n.RUnlock() nsMapRaw, err := json.Marshal(n.NsMap) if err != nil { @@ -54,33 +61,6 @@ func (n *NpmNamespaceCache) MarshalJSON() ([]byte, error) { return nsMapRaw, nil } -type Namespace struct { - name string - LabelsMap map[string]string // Namespace labels -} - -// newNS constructs a new namespace object. -func newNs(name string) *Namespace { - ns := &Namespace{ - name: name, - LabelsMap: make(map[string]string), - } - return ns -} - -func (nsObj *Namespace) appendLabels(newm map[string]string, clear LabelAppendOperation) { - if clear { - nsObj.LabelsMap = make(map[string]string) - } - for k, v := range newm { - nsObj.LabelsMap[k] = v - } -} - -func (nsObj *Namespace) removeLabelsWithKey(key string) { - delete(nsObj.LabelsMap, key) -} - type NamespaceController struct { dp dataplane.GenericDataplane nameSpaceLister corelisters.NamespaceLister @@ -106,6 +86,10 @@ func NewNamespaceController(nameSpaceInformer coreinformer.NamespaceInformer, dp return nameSpaceController } +func (n *NamespaceController) GetCache() map[string]*common.Namespace { + return n.npmNamespaceCache.GetCache() +} + // filter this event if we do not need to handle this event func (nsc *NamespaceController) needSync(obj interface{}, event string) (string, bool) { needSync := false @@ -322,7 +306,7 @@ func (nsc *NamespaceController) syncAddNamespace(nsObj *corev1.Namespace) error namespaceSets := []*ipsets.IPSetMetadata{ipsets.NewIPSetMetadata(nsObj.ObjectMeta.Name, ipsets.Namespace)} setsToAddNamespaceTo := []*ipsets.IPSetMetadata{kubeAllNamespaces} - npmNs := newNs(nsObj.ObjectMeta.Name) + npmNs := common.NewNs(nsObj.ObjectMeta.Name) nsc.npmNamespaceCache.NsMap[nsObj.ObjectMeta.Name] = npmNs // Add the namespace to its label's ipset list. @@ -337,7 +321,7 @@ func (nsc *NamespaceController) syncAddNamespace(nsObj *corev1.Namespace) error setsToAddNamespaceTo = append(setsToAddNamespaceTo, labelIPSets...) // Append succeeded labels to the cache NS obj - npmNs.appendLabels(map[string]string{nsLabelKey: nsLabelVal}, appendToExistingLabels) + npmNs.AppendLabels(map[string]string{nsLabelKey: nsLabelVal}, common.AppendToExistingLabels) } if err := nsc.dp.AddToLists(setsToAddNamespaceTo, namespaceSets); err != nil { @@ -390,7 +374,7 @@ func (nsc *NamespaceController) syncUpdateNamespace(newNsObj *corev1.Namespace) // (TODO) need to remove this ordering dependency removedLabelKey, removedLabelValue := util.GetLabelKVFromSet(nsLabelVal) if removedLabelValue != "" { - curNsObj.removeLabelsWithKey(removedLabelKey) + curNsObj.RemoveLabelsWithKey(removedLabelKey) } } @@ -414,14 +398,14 @@ func (nsc *NamespaceController) syncUpdateNamespace(newNsObj *corev1.Namespace) // only after both ipsets for a given label's key value pair are added successfully addedLabelKey, addedLabelValue := util.GetLabelKVFromSet(nsLabelVal) if addedLabelValue != "" { - curNsObj.appendLabels(map[string]string{addedLabelKey: addedLabelValue}, appendToExistingLabels) + curNsObj.AppendLabels(map[string]string{addedLabelKey: addedLabelValue}, common.AppendToExistingLabels) } } // Append all labels to the cache NS obj // If due to ordering issue the above deleted and added labels are not correct, // this below appendLabels will help ensure correct state in cache for all successful ops. - curNsObj.appendLabels(newNsLabel, clearExistingLabels) + curNsObj.AppendLabels(newNsLabel, common.ClearExistingLabels) nsc.npmNamespaceCache.NsMap[newNsName] = curNsObj return metrics.UpdateOp, nil @@ -458,7 +442,7 @@ func (nsc *NamespaceController) cleanDeletedNamespace(cachedNsKey string) error } // remove labels from the cache NS obj - cachedNsObj.removeLabelsWithKey(nsLabelKey) + cachedNsObj.RemoveLabelsWithKey(nsLabelKey) } allNamespacesSet := ipsets.NewIPSetMetadata(util.KubeAllNamespacesFlag, ipsets.KeyLabelOfNamespace) diff --git a/npm/pkg/controlplane/controllers/v2/namespaceController_test.go b/npm/pkg/controlplane/controllers/v2/namespaceController_test.go index 0d21ab7a51..e6248f52f1 100644 --- a/npm/pkg/controlplane/controllers/v2/namespaceController_test.go +++ b/npm/pkg/controlplane/controllers/v2/namespaceController_test.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/npm/metrics" "github.com/Azure/azure-container-networking/npm/metrics/promutil" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/Azure/azure-container-networking/npm/pkg/dataplane" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets" dpmocks "github.com/Azure/azure-container-networking/npm/pkg/dataplane/mocks" @@ -90,7 +91,7 @@ func (f *nameSpaceFixture) newNsController(_ chan struct{}) { kubeclient := k8sfake.NewSimpleClientset(f.kubeobjects...) f.kubeInformer = kubeinformers.NewSharedInformerFactory(kubeclient, noResyncPeriodFunc()) - npmNamespaceCache := &NpmNamespaceCache{NsMap: make(map[string]*Namespace)} + npmNamespaceCache := &NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)} f.nsController = NewNamespaceController( f.kubeInformer.Core().V1().Namespaces(), f.dp, npmNamespaceCache) @@ -731,10 +732,10 @@ func checkNsTestResult(testName string, f *nameSpaceFixture, testCases []expecte } func TestNSMapMarshalJSON(t *testing.T) { - npmNSCache := &NpmNamespaceCache{NsMap: make(map[string]*Namespace)} + npmNSCache := &NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)} nsName := "ns-test" - ns := &Namespace{ - name: nsName, + ns := &common.Namespace{ + Name: nsName, LabelsMap: map[string]string{ "test-key": "test-value", }, @@ -744,7 +745,7 @@ func TestNSMapMarshalJSON(t *testing.T) { nsMapRaw, err := npmNSCache.MarshalJSON() require.NoError(t, err) - expect := []byte(`{"ns-test":{"LabelsMap":{"test-key":"test-value"}}}`) + expect := []byte(`{"ns-test":{"Name":"ns-test","LabelsMap":{"test-key":"test-value"}}}`) assert.ElementsMatch(t, expect, nsMapRaw) } diff --git a/npm/pkg/controlplane/controllers/v2/networkPolicyController.go b/npm/pkg/controlplane/controllers/v2/networkPolicyController.go index d2ab105d42..17bab4d553 100644 --- a/npm/pkg/controlplane/controllers/v2/networkPolicyController.go +++ b/npm/pkg/controlplane/controllers/v2/networkPolicyController.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "reflect" + "sync" "time" "github.com/Azure/azure-container-networking/npm/metrics" @@ -29,12 +30,19 @@ var ( ) type NetworkPolicyController struct { + sync.RWMutex netPolLister netpollister.NetworkPolicyLister workqueue workqueue.RateLimitingInterface rawNpSpecMap map[string]*networkingv1.NetworkPolicySpec // Key is / dp dataplane.GenericDataplane } +func (c *NetworkPolicyController) GetCache() map[string]*networkingv1.NetworkPolicySpec { + c.RLock() + defer c.RUnlock() + return c.rawNpSpecMap +} + func NewNetworkPolicyController(npInformer networkinginformers.NetworkPolicyInformer, dp dataplane.GenericDataplane) *NetworkPolicyController { netPolController := &NetworkPolicyController{ netPolLister: npInformer.Lister(), diff --git a/npm/pkg/controlplane/controllers/v2/npmcache.go b/npm/pkg/controlplane/controllers/v2/npmcache.go deleted file mode 100644 index 9b9ffe5d93..0000000000 --- a/npm/pkg/controlplane/controllers/v2/npmcache.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2018 Microsoft. All rights reserved. -// MIT License -package controllers - -import ( - "github.com/Azure/azure-container-networking/npm/ipsm" -) - -type Cache struct { - NodeName string - NsMap map[string]*Namespace - PodMap map[string]*NpmPod - ListMap map[string]*ipsm.Ipset - SetMap map[string]*ipsm.Ipset -} diff --git a/npm/pkg/controlplane/controllers/v2/podController.go b/npm/pkg/controlplane/controllers/v2/podController.go index 3524758146..e6b58fff45 100644 --- a/npm/pkg/controlplane/controllers/v2/podController.go +++ b/npm/pkg/controlplane/controllers/v2/podController.go @@ -10,6 +10,7 @@ import ( "time" "github.com/Azure/azure-container-networking/npm/metrics" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/Azure/azure-container-networking/npm/pkg/dataplane" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets" "github.com/Azure/azure-container-networking/npm/util" @@ -17,7 +18,6 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - k8slabels "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" coreinformer "k8s.io/client-go/informers/core/v1" @@ -37,72 +37,12 @@ const ( var kubeAllNamespaces = &ipsets.IPSetMetadata{Name: util.KubeAllNamespacesFlag, Type: ipsets.KeyLabelOfNamespace} -type NpmPod struct { - Name string - Namespace string - PodIP string - Labels map[string]string - ContainerPorts []corev1.ContainerPort - Phase corev1.PodPhase -} - -func newNpmPod(podObj *corev1.Pod) *NpmPod { - return &NpmPod{ - Name: podObj.ObjectMeta.Name, - Namespace: podObj.ObjectMeta.Namespace, - PodIP: podObj.Status.PodIP, - Labels: make(map[string]string), - ContainerPorts: []corev1.ContainerPort{}, - Phase: podObj.Status.Phase, - } -} - -func (nPod *NpmPod) appendLabels(newLabelMap map[string]string, clear LabelAppendOperation) { - if clear { - nPod.Labels = make(map[string]string) - } - for k, v := range newLabelMap { - nPod.Labels[k] = v - } -} - -func (nPod *NpmPod) removeLabelsWithKey(key string) { - delete(nPod.Labels, key) -} - -func (nPod *NpmPod) appendContainerPorts(podObj *corev1.Pod) { - nPod.ContainerPorts = getContainerPortList(podObj) -} - -func (nPod *NpmPod) removeContainerPorts() { - nPod.ContainerPorts = []corev1.ContainerPort{} -} - -// This function can be expanded to other attribs if needed -func (nPod *NpmPod) updateNpmPodAttributes(podObj *corev1.Pod) { - if nPod.Phase != podObj.Status.Phase { - nPod.Phase = podObj.Status.Phase - } -} - -// noUpdate evaluates whether NpmPod is required to be update given podObj. -func (nPod *NpmPod) noUpdate(podObj *corev1.Pod) bool { - return nPod.Namespace == podObj.ObjectMeta.Namespace && - nPod.Name == podObj.ObjectMeta.Name && - nPod.Phase == podObj.Status.Phase && - nPod.PodIP == podObj.Status.PodIP && - k8slabels.Equals(nPod.Labels, podObj.ObjectMeta.Labels) && - // TODO(jungukcho) to avoid using DeepEqual for ContainerPorts, - // it needs a precise sorting. Will optimize it later if needed. - reflect.DeepEqual(nPod.ContainerPorts, getContainerPortList(podObj)) -} - type PodController struct { podLister corelisters.PodLister workqueue workqueue.RateLimitingInterface dp dataplane.GenericDataplane - podMap map[string]*NpmPod // Key is / - sync.Mutex + podMap map[string]*common.NpmPod // Key is / + sync.RWMutex npmNamespaceCache *NpmNamespaceCache } @@ -111,7 +51,7 @@ func NewPodController(podInformer coreinformer.PodInformer, dp dataplane.Generic podLister: podInformer.Lister(), workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Pods"), dp: dp, - podMap: make(map[string]*NpmPod), + podMap: make(map[string]*common.NpmPod), npmNamespaceCache: npmNamespaceCache, } @@ -372,7 +312,7 @@ func (c *PodController) syncPod(key string) error { // if pod does not have different states against lastly applied states stored in cachedNpmPod, // podController does not need to reconcile this update. // in this updatePod event, newPod was updated with states which PodController does not need to reconcile. - if cachedNpmPod.noUpdate(pod) { + if cachedNpmPod.NoUpdate(pod) { return nil } } @@ -410,7 +350,7 @@ func (c *PodController) syncAddedPod(podObj *corev1.Pod) error { } // Create npmPod and add it to the podMap - npmPodObj := newNpmPod(podObj) + npmPodObj := common.NewNpmPod(podObj) c.podMap[podKey] = npmPodObj // Get lists of podLabelKey and podLabelKey + podLavelValue ,and then start adding them to ipsets. @@ -426,16 +366,16 @@ func (c *PodController) syncAddedPod(podObj *corev1.Pod) error { if err = c.dp.AddToSets(allSets, podMetadata); err != nil { return fmt.Errorf("[syncAddedPod] Error: failed to add pod to label ipset with err: %w", err) } - npmPodObj.appendLabels(map[string]string{labelKey: labelVal}, appendToExistingLabels) + npmPodObj.AppendLabels(map[string]string{labelKey: labelVal}, common.AppendToExistingLabels) } // Add pod's named ports from its ipset. klog.Infof("Adding named port ipsets") - containerPorts := getContainerPortList(podObj) + containerPorts := common.GetContainerPortList(podObj) if err = c.manageNamedPortIpsets(containerPorts, podKey, npmPodObj.PodIP, podObj.Spec.NodeName, addNamedPort); err != nil { return fmt.Errorf("[syncAddedPod] Error: failed to add pod to named port ipset with err: %w", err) } - npmPodObj.appendContainerPorts(podObj) + npmPodObj.AppendContainerPorts(podObj) return nil } @@ -458,7 +398,7 @@ func (c *PodController) syncAddAndUpdatePod(newPodObj *corev1.Pod) (metrics.Oper } // Add namespace object into NsMap cache only when two ipset operations are successful. - npmNs := newNs(newPodObj.Namespace) + npmNs := common.NewNs(newPodObj.Namespace) c.npmNamespaceCache.NsMap[newPodObj.Namespace] = npmNs } c.npmNamespaceCache.Unlock() @@ -518,7 +458,7 @@ func (c *PodController) syncAddAndUpdatePod(newPodObj *corev1.Pod) (metrics.Oper // key + val ipsets are worked on. 0th index will be key and 1st index will be value of the label removedLabelKey, removedLabelValue := util.GetLabelKVFromSet(removeIPSetName) if removedLabelValue != "" { - cachedNpmPod.removeLabelsWithKey(removedLabelKey) + cachedNpmPod.RemoveLabelsWithKey(removedLabelKey) } } @@ -543,18 +483,18 @@ func (c *PodController) syncAddAndUpdatePod(newPodObj *corev1.Pod) (metrics.Oper // (TODO) will need to remove this ordering dependency addedLabelKey, addedLabelValue := util.GetLabelKVFromSet(addIPSetName) if addedLabelValue != "" { - cachedNpmPod.appendLabels(map[string]string{addedLabelKey: addedLabelValue}, appendToExistingLabels) + cachedNpmPod.AppendLabels(map[string]string{addedLabelKey: addedLabelValue}, common.AppendToExistingLabels) } } // This will ensure after all labels are worked on to overwrite. This way will reduce any bugs introduced above // If due to ordering issue the above deleted and added labels are not correct, // this below appendLabels will help ensure correct state in cache for all successful ops. - cachedNpmPod.appendLabels(newPodObj.Labels, clearExistingLabels) + cachedNpmPod.AppendLabels(newPodObj.Labels, common.ClearExistingLabels) // (TODO): optimize named port addition and deletions. // named ports are mostly static once configured in todays usage pattern // so keeping this simple by deleting all and re-adding - newPodPorts := getContainerPortList(newPodObj) + newPodPorts := common.GetContainerPortList(newPodObj) if !reflect.DeepEqual(cachedNpmPod.ContainerPorts, newPodPorts) { // Delete cached pod's named ports from its ipset. if err = c.manageNamedPortIpsets( @@ -562,15 +502,15 @@ func (c *PodController) syncAddAndUpdatePod(newPodObj *corev1.Pod) (metrics.Oper return metrics.UpdateOp, fmt.Errorf("[syncAddAndUpdatePod] Error: failed to delete pod from named port ipset with err: %w", err) } // Since portList ipset deletion is successful, NPM can remove cachedContainerPorts - cachedNpmPod.removeContainerPorts() + cachedNpmPod.RemoveContainerPorts() // Add new pod's named ports from its ipset. if err = c.manageNamedPortIpsets(newPodPorts, podKey, newPodObj.Status.PodIP, newPodObj.Spec.NodeName, addNamedPort); err != nil { return metrics.UpdateOp, fmt.Errorf("[syncAddAndUpdatePod] Error: failed to add pod to named port ipset with err: %w", err) } - cachedNpmPod.appendContainerPorts(newPodObj) + cachedNpmPod.AppendContainerPorts(newPodObj) } - cachedNpmPod.updateNpmPodAttributes(newPodObj) + cachedNpmPod.UpdateNpmPodAttributes(newPodObj) return metrics.UpdateOp, nil } @@ -606,7 +546,7 @@ func (c *PodController) cleanUpDeletedPod(cachedNpmPodKey string) error { cachedPodMetadata); err != nil { return fmt.Errorf("[cleanUpDeletedPod] Error: failed to delete pod from label ipset with err: %w", err) } - cachedNpmPod.removeLabelsWithKey(labelKey) + cachedNpmPod.RemoveLabelsWithKey(labelKey) } // Delete pod's named ports from its ipset. Need to pass true in the manageNamedPortIpsets function call @@ -685,11 +625,3 @@ func hasValidPodIP(podObj *corev1.Pod) bool { func isHostNetworkPod(podObj *corev1.Pod) bool { return podObj.Spec.HostNetwork } - -func getContainerPortList(podObj *corev1.Pod) []corev1.ContainerPort { - portList := []corev1.ContainerPort{} - for i := range podObj.Spec.Containers { - portList = append(portList, podObj.Spec.Containers[i].Ports...) - } - return portList -} diff --git a/npm/pkg/controlplane/controllers/v2/podController_test.go b/npm/pkg/controlplane/controllers/v2/podController_test.go index 975d6ddd6f..5beb3e592f 100644 --- a/npm/pkg/controlplane/controllers/v2/podController_test.go +++ b/npm/pkg/controlplane/controllers/v2/podController_test.go @@ -10,6 +10,7 @@ import ( "github.com/Azure/azure-container-networking/npm/metrics" "github.com/Azure/azure-container-networking/npm/metrics/promutil" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/Azure/azure-container-networking/npm/pkg/dataplane" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets" dpmocks "github.com/Azure/azure-container-networking/npm/pkg/dataplane/mocks" @@ -73,7 +74,7 @@ func (f *podFixture) newPodController(_ chan struct{}) { kubeclient := k8sfake.NewSimpleClientset(f.kubeobjects...) f.kubeInformer = kubeinformers.NewSharedInformerFactory(kubeclient, noResyncPeriodFunc()) - npmNamespaceCache := &NpmNamespaceCache{NsMap: make(map[string]*Namespace)} + npmNamespaceCache := &NpmNamespaceCache{NsMap: make(map[string]*common.Namespace)} f.podController = NewPodController(f.kubeInformer.Core().V1().Pods(), f.dp, npmNamespaceCache) for _, pod := range f.podLister { @@ -246,7 +247,7 @@ func checkNpmPodWithInput(testName string, f *podFixture, inputPodObj *corev1.Po f.t.Errorf("%s failed @ Labels check got = %v, want %v", testName, cachedNpmPodObj.Labels, inputPodObj.Labels) } - inputPortList := getContainerPortList(inputPodObj) + inputPortList := common.GetContainerPortList(inputPodObj) if !reflect.DeepEqual(cachedNpmPodObj.ContainerPorts, inputPortList) { f.t.Errorf("%s failed @ Container port check got = %v, want %v", testName, cachedNpmPodObj.PodIP, inputPortList) } @@ -754,7 +755,7 @@ func TestPodMapMarshalJSON(t *testing.T) { podKey, err := cache.MetaNamespaceKeyFunc(pod) assert.NoError(t, err) - npmPod := newNpmPod(pod) + npmPod := common.NewNpmPod(pod) f.podController.podMap[podKey] = npmPod npMapRaw, err := f.podController.MarshalJSON() @@ -950,13 +951,13 @@ func TestNPMPodNoUpdate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() corev1Pod := createPod(tt.podName, tt.ns, tt.rv, tt.podIP, tt.labels, tt.isHostNetwork, tt.podPhase) - npmPod := newNpmPod(corev1Pod) + npmPod := common.NewNpmPod(corev1Pod) if tt.updatingNPMPod { - npmPod.appendLabels(corev1Pod.Labels, appendToExistingLabels) - npmPod.updateNpmPodAttributes(corev1Pod) - npmPod.appendContainerPorts(corev1Pod) + npmPod.AppendLabels(corev1Pod.Labels, common.AppendToExistingLabels) + npmPod.UpdateNpmPodAttributes(corev1Pod) + npmPod.AppendContainerPorts(corev1Pod) } - noUpdate := npmPod.noUpdate(corev1Pod) + noUpdate := npmPod.NoUpdate(corev1Pod) require.Equal(t, tt.expectedNoUpdate, noUpdate) }) } diff --git a/npm/pkg/controlplane/goalstateprocessor/goalstateprocessor.go b/npm/pkg/controlplane/goalstateprocessor/goalstateprocessor.go index 7753eccb96..d3a34e9283 100644 --- a/npm/pkg/controlplane/goalstateprocessor/goalstateprocessor.go +++ b/npm/pkg/controlplane/goalstateprocessor/goalstateprocessor.go @@ -174,10 +174,18 @@ func (gsp *GoalStateProcessor) processHydrationEvent(payload map[string]*protos. } cachedIPSetNames := gsp.dp.GetAllIPSets() + hashedsetnames := make([]string, len(cachedIPSetNames)) + toDeleteIPSets := make([]string, 0) + i := 0 + for name := range cachedIPSetNames { + hashedsetnames[i] = name + i++ + } + if appendedIPSets == nil { - toDeleteIPSets = cachedIPSetNames + toDeleteIPSets = hashedsetnames } else { for _, ipset := range cachedIPSetNames { if _, ok := appendedIPSets[ipset]; !ok { diff --git a/npm/pkg/dataplane/dataplane.go b/npm/pkg/dataplane/dataplane.go index 2172aa9972..fe0997bec0 100644 --- a/npm/pkg/dataplane/dataplane.go +++ b/npm/pkg/dataplane/dataplane.go @@ -301,7 +301,7 @@ func (dp *DataPlane) UpdatePolicy(policy *policies.NPMNetworkPolicy) error { return nil } -func (dp *DataPlane) GetAllIPSets() []string { +func (dp *DataPlane) GetAllIPSets() map[string]string { return dp.ipsetMgr.GetAllIPSets() } diff --git a/npm/pkg/dataplane/debug/const.go b/npm/pkg/dataplane/debug/const.go index fe36085c27..d2f9a26ab5 100644 --- a/npm/pkg/dataplane/debug/const.go +++ b/npm/pkg/dataplane/debug/const.go @@ -1,7 +1,6 @@ -package dataplane +package debug import ( - "errors" "regexp" ) @@ -31,17 +30,11 @@ var AzureNPMChains = []string{ var matcher = regexp.MustCompile(`(?i)[^ ]+-in-ns-[^ ]+-\d(out|in)\b`) -// error type -var ( - errSetNotExist = errors.New("set does not exists") - errInvalidIPAddress = errors.New("invalid ipaddress, no equivalent pod found") - errInvalidInput = errors.New("invalid input") - errSetType = errors.New("invalid set type") -) - // To test paser, converter, and trafficAnalyzer with stored files. const ( - iptableSaveFile = "../testdata/iptablesave" + iptableSaveFileV1 = "../testdata/iptablesave-v1" + iptableSaveFileV2 = "../testdata/iptablesave-v2" // stored file with json compatible form (i.e., can call json.Unmarshal) - npmCacheFile = "../testdata/npmcache.json" + npmCacheFileV1 = "../testdata/npmcachev1.json" + npmCacheFileV2 = "../testdata/npmcachev2.json" ) diff --git a/npm/pkg/dataplane/debug/converter.go b/npm/pkg/dataplane/debug/converter.go index ac9be7ad51..e8e27a961c 100644 --- a/npm/pkg/dataplane/debug/converter.go +++ b/npm/pkg/dataplane/debug/converter.go @@ -1,4 +1,4 @@ -package dataplane +package debug import ( "bytes" @@ -13,21 +13,37 @@ import ( "strconv" "strings" + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/npm/http/api" - controllersv1 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v1" + npmcommon "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" + "github.com/Azure/azure-container-networking/npm/pkg/dataplane/ipsets" NPMIPtable "github.com/Azure/azure-container-networking/npm/pkg/dataplane/iptables" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/parse" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/pb" + "github.com/Azure/azure-container-networking/npm/pkg/models" "github.com/Azure/azure-container-networking/npm/util" - "google.golang.org/protobuf/encoding/protojson" + "github.com/pkg/errors" +) + +var ( + ErrUnknownSetType = fmt.Errorf("unknown set type") + EgressChain = "AZURE-NPM-EGRESS" + EgressChainPrefix = EgressChain + "-" + + IngressChain = "AZURE-NPM-INGRESS" + IngressChainPrefix = IngressChain + "-" ) // Converter struct type Converter struct { - ListMap map[string]string // key: hash(value), value: one of namespace, label of namespace, multiple values - SetMap map[string]string // key: hash(value), value: one of label of pods, cidr, namedport - AzureNPMChains map[string]bool - NPMCache *controllersv1.Cache + NPMDebugEndpointHost string + NPMDebugEndpointPort string + Parser parse.IPTablesParser + ListMap map[string]string // key: hash(value), value: one of namespace, label of namespace, multiple values + SetMap map[string]string // key: hash(value), value: one of label of pods, cidr, namedport + AzureNPMChains map[string]bool + NPMCache npmcommon.GenericCache + EnableV2NPM bool } // NpmCacheFromFile initialize NPM cache from file. @@ -37,11 +53,11 @@ func (c *Converter) NpmCacheFromFile(npmCacheJSONFile string) error { return fmt.Errorf("failed to read %s file : %w", npmCacheJSONFile, err) } - c.NPMCache = &controllersv1.Cache{} - err = json.Unmarshal(byteArray, c.NPMCache) + err = c.getCacheFromBytes(byteArray) if err != nil { - return fmt.Errorf("failed to unmarshal %s due to %w", string(byteArray), err) + return errors.Wrap(err, "failed to get cache from file") } + return nil } @@ -50,13 +66,14 @@ func (c *Converter) NpmCache() error { req, err := http.NewRequestWithContext( context.Background(), http.MethodGet, - fmt.Sprintf("http://localhost:%v%v", api.DefaultHttpPort, api.NPMMgrPath), + fmt.Sprintf("%v:%v%v", c.NPMDebugEndpointHost, c.NPMDebugEndpointPort, api.NPMMgrPath), nil, ) if err != nil { return fmt.Errorf("failed to create http request : %w", err) } - resp, err := http.DefaultClient.Do(req) + client := &http.Client{} + resp, err := client.Do(req) if err != nil { return fmt.Errorf("failed to request NPM Cache : %w", err) } @@ -65,11 +82,66 @@ func (c *Converter) NpmCache() error { if err != nil { return fmt.Errorf("failed to read response's data : %w", err) } - c.NPMCache = &controllersv1.Cache{} - err = json.Unmarshal(byteArray, c.NPMCache) + + err = c.getCacheFromBytes(byteArray) if err != nil { - return fmt.Errorf("failed to unmarshal %s due to %w", string(byteArray), err) + return errors.Wrapf(err, "failed to get cache from debug http endpoint") } + + return nil +} + +// Hello Time Traveler: +// This is the client end of the debug dragons den. For issues related to marshaling, +// please refer to the custom marshaling that happens in npm/npm.go +// best of luck +func (c *Converter) getCacheFromBytes(byteArray []byte) error { + m := map[models.CacheKey]json.RawMessage{} + if c.EnableV2NPM { + cache := &npmcommon.Cache{} + if err := json.Unmarshal(byteArray, &m); err != nil { + return errors.Wrapf(err, "failed to unmarshal into v2 cache map") + } + + if err := json.Unmarshal(m[models.NsMap], &cache.NsMap); err != nil { + return errors.Wrapf(err, "failed to unmarshal nsmap into v2 cache") + } + + if err := json.Unmarshal(m[models.PodMap], &cache.PodMap); err != nil { + return errors.Wrapf(err, "failed to unmarshal podmap into v2 cache") + } + + if err := json.Unmarshal(m[models.SetMap], &cache.SetMap); err != nil { + return errors.Wrapf(err, "failed to unmarshal setmap into v2 cache") + } + + c.NPMCache = cache + + } else { + cache := &npmcommon.Cache{} + if err := json.Unmarshal(byteArray, &m); err != nil { + return errors.Wrapf(err, "failed to unmarshal into v1 cache map") + } + + if err := json.Unmarshal(m[models.NsMap], &cache.NsMap); err != nil { + return errors.Wrapf(err, "failed to unmarshal nsmap into v1 cache map") + } + + if err := json.Unmarshal(m[models.PodMap], &cache.PodMap); err != nil { + return errors.Wrapf(err, "failed to unmarshal podmap into v1 cache") + } + + if err := json.Unmarshal(m[models.SetMap], &cache.SetMap); err != nil { + return errors.Wrapf(err, "failed to unmarshal setmap into v1 cache") + } + + if err := json.Unmarshal(m[models.ListMap], &cache.ListMap); err != nil { + return errors.Wrapf(err, "failed to unmarshal listmap into v1 cache") + } + + c.NPMCache = cache + } + return nil } @@ -77,20 +149,24 @@ func (c *Converter) NpmCache() error { func (c *Converter) initConverterFile(npmCacheJSONFile string) error { err := c.NpmCacheFromFile(npmCacheJSONFile) if err != nil { - return fmt.Errorf("error occurred during initialize converter : %w", err) + return fmt.Errorf("error occurred during initialize converter from file: %w", err) } c.initConverterMaps() return nil } // Initialize converter from node. -func (c *Converter) initConverter() error { +func (c *Converter) InitConverter() error { err := c.NpmCache() if err != nil { return fmt.Errorf("error occurred during initialize converter : %w", err) } c.initConverterMaps() + c.Parser = parse.IPTablesParser{ + IOShim: common.NewIOShim(), + } + return nil } @@ -100,42 +176,23 @@ func (c *Converter) initConverterMaps() { for _, chain := range AzureNPMChains { c.AzureNPMChains[chain] = true } - c.ListMap = make(map[string]string) - c.SetMap = make(map[string]string) - for k := range c.NPMCache.ListMap { - hashedName := util.GetHashedName(k) - c.ListMap[hashedName] = k - } - for k := range c.NPMCache.SetMap { - hashedName := util.GetHashedName(k) - c.SetMap[hashedName] = k - } + c.ListMap = c.NPMCache.GetListMap() + c.SetMap = c.NPMCache.GetSetMap() } -// GetJSONRulesFromIptableFile returns a list of json rules from npmCache and iptable-save files. -func (c *Converter) GetJSONRulesFromIptableFile( - tableName string, - npmCacheFile string, - iptableSaveFile string, -) ([][]byte, error) { - - pbRule, err := c.GetProtobufRulesFromIptableFile(tableName, npmCacheFile, iptableSaveFile) - if err != nil { - return nil, fmt.Errorf("error occurred during getting JSON rules from iptables : %w", err) - } - return c.jsonRuleList(pbRule) -} - -// GetJSONRulesFromIptables returns a list of json rules from node -func (c *Converter) GetJSONRulesFromIptables(tableName string) ([][]byte, error) { - pbRule, err := c.GetProtobufRulesFromIptable(tableName) - if err != nil { - return nil, fmt.Errorf("error occurred during getting JSON rules from iptables : %w", err) +func (c *Converter) isAzureNPMChain(chain string) bool { + if c.EnableV2NPM { + if strings.HasPrefix(chain, "AZURE-NPM") { + return true + } + } else { + return c.AzureNPMChains[chain] } - return c.jsonRuleList(pbRule) + return false } +/* // Convert list of protobuf rules to list of JSON rules. func (c *Converter) jsonRuleList(pbRules []*pb.RuleResponse) ([][]byte, error) { ruleResListJSON := make([][]byte, 0) @@ -152,17 +209,18 @@ func (c *Converter) jsonRuleList(pbRules []*pb.RuleResponse) ([][]byte, error) { } return ruleResListJSON, nil } +*/ // GetProtobufRulesFromIptableFile returns a list of protobuf rules from npmCache and iptable-save files. func (c *Converter) GetProtobufRulesFromIptableFile( tableName string, npmCacheFile string, iptableSaveFile string, -) ([]*pb.RuleResponse, error) { +) (map[*pb.RuleResponse]struct{}, error) { err := c.initConverterFile(npmCacheFile) if err != nil { - return nil, fmt.Errorf("error occurred during getting protobuf rules from iptables : %w", err) + return nil, fmt.Errorf("error occurred during getting protobuf rules from iptables from file: %w", err) } ipTable, err := parse.IptablesFile(tableName, iptableSaveFile) @@ -171,15 +229,15 @@ func (c *Converter) GetProtobufRulesFromIptableFile( } ruleResList, err := c.pbRuleList(ipTable) if err != nil { - return nil, fmt.Errorf("error occurred during getting protobuf rules from iptables : %w", err) + return nil, fmt.Errorf("error occurred during getting protobuf rules from iptables pb rule list: %w", err) } return ruleResList, nil } // GetProtobufRulesFromIptable returns a list of protobuf rules from node. -func (c *Converter) GetProtobufRulesFromIptable(tableName string) ([]*pb.RuleResponse, error) { - err := c.initConverter() +func (c *Converter) GetProtobufRulesFromIptable(tableName string) (map[*pb.RuleResponse]struct{}, error) { + err := c.InitConverter() if err != nil { return nil, fmt.Errorf("error occurred during getting protobuf rules from iptables : %w", err) } @@ -188,6 +246,7 @@ func (c *Converter) GetProtobufRulesFromIptable(tableName string) ([]*pb.RuleRes if err != nil { return nil, fmt.Errorf("error occurred during parsing iptables : %w", err) } + ruleResList, err := c.pbRuleList(ipTable) if err != nil { return nil, fmt.Errorf("error occurred during getting protobuf rules from iptables : %w", err) @@ -197,46 +256,123 @@ func (c *Converter) GetProtobufRulesFromIptable(tableName string) ([]*pb.RuleRes } // Create a list of protobuf rules from iptable. -func (c *Converter) pbRuleList(ipTable *NPMIPtable.Table) ([]*pb.RuleResponse, error) { - ruleResList := make([]*pb.RuleResponse, 0) +func (c *Converter) pbRuleList(ipTable *NPMIPtable.Table) (map[*pb.RuleResponse]struct{}, error) { + allRulesInNPMChains := make(map[*pb.RuleResponse]struct{}, 0) + + // iterate through all chains in the filter table for _, v := range ipTable.Chains { - chainRules, err := c.getRulesFromChain(v) - if err != nil { - return nil, fmt.Errorf("error occurred during getting protobuf rule list : %w", err) + if c.isAzureNPMChain(v.Name) { + + // can skip this chain in V2 since it's an accept + if c.EnableV2NPM && (strings.HasPrefix(v.Name, "AZURE-NPM-INGRESS-ALLOW-MARK") || (strings.HasPrefix(v.Name, "AZURE-NPM-ACCEPT"))) { + continue + } + + rulesFromChain, err := c.getRulesFromChain(v) + if err != nil { + return nil, fmt.Errorf("error occurred during getting protobuf rule list : %w", err) + } + /* + if strings.HasPrefix("AZURE-NPM-EGRESS") { + for i := range rulesFromChain { + rulesFromChain[i].SrcList = + } + } + */ + for _, rule := range rulesFromChain { + allRulesInNPMChains[rule] = struct{}{} + } } - ruleResList = append(ruleResList, chainRules...) } - return ruleResList, nil + if c.EnableV2NPM { + parentRules := make([]*pb.RuleResponse, 0) + for childRule := range allRulesInNPMChains { + + // if rule is a string-int, we need to find the parent jump + // to add the src for egress and dst for ingress + if strings.HasPrefix(childRule.Chain, EgressChainPrefix) { + for parentRule := range allRulesInNPMChains { + if strings.HasPrefix(parentRule.Chain, EgressChain) && parentRule.JumpTo == childRule.Chain { + childRule.SrcList = append(childRule.SrcList, parentRule.SrcList...) + childRule.Comment = parentRule.Comment + parentRules = append(parentRules, parentRule) + } + } + } + if strings.HasPrefix(childRule.Chain, IngressChainPrefix) { + for parentRule := range allRulesInNPMChains { + if strings.HasPrefix(parentRule.Chain, IngressChain) && parentRule.JumpTo == childRule.Chain { + childRule.DstList = append(childRule.DstList, parentRule.DstList...) + childRule.Comment = parentRule.Comment + parentRules = append(parentRules, parentRule) + } + } + } + } + for _, parentRule := range parentRules { + delete(allRulesInNPMChains, parentRule) + } + } + + return allRulesInNPMChains, nil } func (c *Converter) getRulesFromChain(iptableChain *NPMIPtable.Chain) ([]*pb.RuleResponse, error) { rules := make([]*pb.RuleResponse, 0) + // loop through each chain, if it has a jump, follow that jump + // loop through rules in that jumped chain + for _, v := range iptableChain.Rules { rule := &pb.RuleResponse{} rule.Chain = iptableChain.Name - // filter other chains except for Azure NPM specific chains. - if _, ok := c.AzureNPMChains[rule.Chain]; !ok { - continue - } rule.Protocol = v.Protocol - switch v.Target.Name { - case util.IptablesMark: - rule.Allowed = true - case util.IptablesDrop: - rule.Allowed = false - default: - // ignore other targets - continue + + if c.EnableV2NPM { + // chain name has to end in hash np for it to determine if allow or drop + // ignore jumps from parent AZURE-NPM + switch v.Target.Name { + case util.IptablesAzureIngressAllowMarkChain: + rule.Allowed = true + + case util.IptablesAzureAcceptChain: + rule.Allowed = true + default: + // ignore other targets + rule.Allowed = false + } + } else { + switch v.Target.Name { + case util.IptablesMark: + rule.Allowed = true + case util.IptablesDrop: + rule.Allowed = false + default: + // ignore other targets + continue + } } + rule.Direction = c.getRuleDirection(iptableChain.Name) err := c.getModulesFromRule(v.Modules, rule) if err != nil { return nil, fmt.Errorf("error occurred during getting rules from chain : %w", err) } + + if v.Target != nil { + rule.JumpTo = v.Target.Name + } + + /* + for _, module := range v.Modules { + if module.Verb + } + */ + rules = append(rules, rule) } + return rules, nil } @@ -275,10 +411,43 @@ func (c *Converter) getSetType(name string, m string) pb.SetType { return pb.SetType_KEYLABELOFPOD } +func (c *Converter) getSetTypeV2(name string) (pb.SetType, ipsets.SetKind) { + var settype pb.SetType + var setmetadata ipsets.IPSetMetadata + + switch { + case strings.HasPrefix(name, util.CIDRPrefix): + settype = pb.SetType_CIDRBLOCKS + setmetadata.Type = ipsets.CIDRBlocks + case strings.HasPrefix(name, util.NamespacePrefix): + settype = pb.SetType_NAMESPACE + setmetadata.Type = ipsets.Namespace + case strings.HasPrefix(name, util.NamedPortIPSetPrefix): + settype = pb.SetType_NAMEDPORTS + setmetadata.Type = ipsets.NamedPorts + case strings.HasPrefix(name, util.PodLabelPrefix): + settype = pb.SetType_KEYLABELOFPOD // could also be KeyValueLabelOfPod + setmetadata.Type = ipsets.KeyLabelOfPod + case strings.HasPrefix(name, util.NamespaceLabelPrefix): + settype = pb.SetType_KEYLABELOFNAMESPACE + setmetadata.Type = ipsets.KeyLabelOfNamespace + case strings.HasPrefix(name, util.NestedLabelPrefix): + settype = pb.SetType_NESTEDLABELOFPOD + setmetadata.Type = ipsets.NestedLabelOfPod + default: + log.Printf("set [%s] unknown settype", name) + settype = pb.SetType_UNKNOWN + setmetadata.Type = ipsets.UnknownType + } + + return settype, setmetadata.GetSetKind() +} + func (c *Converter) getModulesFromRule(moduleList []*NPMIPtable.Module, ruleRes *pb.RuleResponse) error { ruleRes.SrcList = make([]*pb.RuleResponse_SetInfo, 0) ruleRes.DstList = make([]*pb.RuleResponse_SetInfo, 0) ruleRes.UnsortedIpset = make(map[string]string) + for _, module := range moduleList { switch module.Verb { case "set": @@ -289,6 +458,7 @@ func (c *Converter) getModulesFromRule(moduleList []*NPMIPtable.Module, ruleRes case "match-set": setInfo := &pb.RuleResponse_SetInfo{} + // will populate the setinfo and add to ruleRes err := c.populateSetInfo(setInfo, values, ruleRes) if err != nil { return fmt.Errorf("error occurred during getting modules from rules : %w", err) @@ -297,6 +467,8 @@ func (c *Converter) getModulesFromRule(moduleList []*NPMIPtable.Module, ruleRes case "not-match-set": setInfo := &pb.RuleResponse_SetInfo{} + + // will populate the setinfo and add to ruleRes err := c.populateSetInfo(setInfo, values, ruleRes) if err != nil { return fmt.Errorf("error occurred during getting modules from rules : %w", err) @@ -320,6 +492,8 @@ func (c *Converter) getModulesFromRule(moduleList []*NPMIPtable.Module, ruleRes ruleRes.SPort = int32(portNum) } } + case util.IptablesCommentModuleFlag: + ruleRes.Comment = fmt.Sprintf("%+v", module.OptionValueMap[util.IptablesCommentModuleFlag]) default: continue } @@ -327,26 +501,33 @@ func (c *Converter) getModulesFromRule(moduleList []*NPMIPtable.Module, ruleRes return nil } -func (c *Converter) populateSetInfo( - setInfo *pb.RuleResponse_SetInfo, - values []string, - ruleRes *pb.RuleResponse, -) error { +func (c *Converter) populateSetInfo(setInfo *pb.RuleResponse_SetInfo, values []string, ruleRes *pb.RuleResponse) error { ipsetHashedName := values[0] ipsetOrigin := values[1] setInfo.HashedSetName = ipsetHashedName - if v, ok := c.ListMap[ipsetHashedName]; ok { - setInfo.Name = v - setInfo.Type = c.getSetType(v, "ListMap") - } else if v, ok := c.SetMap[ipsetHashedName]; ok { - setInfo.Name = v - setInfo.Type = c.getSetType(v, "SetMap") - if setInfo.Type == pb.SetType_CIDRBLOCKS { - populateCIDRBlockSet(setInfo) + + if c.EnableV2NPM { + setInfo.Name = c.SetMap[ipsetHashedName] + settype, _ := c.getSetTypeV2(setInfo.Name) + if settype == pb.SetType_UNKNOWN { + return errors.Wrapf(ErrUnknownSetType, "unknown set type for set: %s", setInfo.Name) } + + setInfo.Type = settype } else { - return fmt.Errorf("%w : %v", errSetNotExist, ipsetHashedName) + if v, ok := c.ListMap[ipsetHashedName]; ok { + setInfo.Name = v + setInfo.Type = c.getSetType(v, "ListMap") + } else if v, ok := c.SetMap[ipsetHashedName]; ok { + setInfo.Name = v + setInfo.Type = c.getSetType(v, "SetMap") + if setInfo.Type == pb.SetType_CIDRBLOCKS { + populateCIDRBlockSet(setInfo) + } + } else { + return fmt.Errorf("%w : %v", npmcommon.ErrSetNotExist, ipsetHashedName) + } } if len(ipsetOrigin) > MinUnsortedIPSetLength { diff --git a/npm/pkg/dataplane/debug/converter_test.go b/npm/pkg/dataplane/debug/converter_test.go index a94e0687b6..0253f8a6bc 100644 --- a/npm/pkg/dataplane/debug/converter_test.go +++ b/npm/pkg/dataplane/debug/converter_test.go @@ -1,44 +1,119 @@ -package dataplane +package debug import ( - "reflect" + "log" "testing" + "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" NPMIPtable "github.com/Azure/azure-container-networking/npm/pkg/dataplane/iptables" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/pb" "github.com/Azure/azure-container-networking/npm/util" "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" ) -func TestGetJSONRulesFromIptableFile(t *testing.T) { +func TestGetProtobufRulesFromIptableFile(t *testing.T) { c := &Converter{} - _, err := c.GetJSONRulesFromIptableFile( + _, err := c.GetProtobufRulesFromIptableFile( util.IptablesFilterTable, - npmCacheFile, - iptableSaveFile, + npmCacheFileV1, + iptableSaveFileV1, ) if err != nil { - t.Errorf("failed to test GetJSONRulesFromIptable : %v", err) + t.Errorf("error during TestGetJSONRulesFromIptable : %v", err) } } -func TestGetProtobufRulesFromIptableFile(t *testing.T) { - c := &Converter{} - _, err := c.GetProtobufRulesFromIptableFile( +func TestGetProtobufRulesFromIptableFileV2(t *testing.T) { + c := &Converter{ + EnableV2NPM: true, + } + rules, err := c.GetProtobufRulesFromIptableFile( util.IptablesFilterTable, - npmCacheFile, - iptableSaveFile, + npmCacheFileV2, + iptableSaveFileV2, ) + require.NoError(t, err) + log.Printf("rules %+v", rules) + + srcPod := &common.NpmPod{ + Name: "a", + Namespace: "y", + PodIP: "10.224.0.70", + Labels: map[string]string{ + "pod": "a", + }, + ContainerPorts: []v1.ContainerPort{ + { + Name: "serve-80-tcp", + ContainerPort: 80, + Protocol: "TCP", + }, + { + Name: "serve-80-udp", + ContainerPort: 80, + Protocol: "UDP", + }, + { + Name: "serve-81-tcp", + ContainerPort: 81, + Protocol: "TCP", + }, + { + Name: "serve-81-UDP", + ContainerPort: 81, + Protocol: "UDP", + }, + }, + } + + dstPod := &common.NpmPod{ + Name: "a", + Namespace: "y", + PodIP: "10.224.0.70", + Labels: map[string]string{ + "pod": "a", + }, + ContainerPorts: []v1.ContainerPort{ + { + Name: "serve-80-tcp", + ContainerPort: 80, + Protocol: "TCP", + }, + { + Name: "serve-80-udp", + ContainerPort: 80, + Protocol: "UDP", + }, + { + Name: "serve-81-tcp", + ContainerPort: 81, + Protocol: "TCP", + }, + { + Name: "serve-81-UDP", + ContainerPort: 81, + Protocol: "UDP", + }, + }, + } + + hitrules, _, _, err := getHitRules(srcPod, dstPod, rules, c.NPMCache) + require.NoError(t, err) + log.Printf("hitrules %+v", hitrules) if err != nil { - t.Errorf("error during TestGetJSONRulesFromIptable : %v", err) + t.Errorf("failed to test GetJSONRulesFromIptable : %v", err) } } func TestNpmCacheFromFile(t *testing.T) { - c := &Converter{} - err := c.NpmCacheFromFile(npmCacheFile) + c := &Converter{ + EnableV2NPM: true, + } + err := c.NpmCacheFromFile(npmCacheFileV2) if err != nil { - t.Errorf("Failed to decode NPMCache from %s file : %v", npmCacheFile, err) + t.Errorf("Failed to decode NPMCache from %s file : %v", npmCacheFileV1, err) } } @@ -90,8 +165,10 @@ func TestGetSetType(t *testing.T) { }, } - c := &Converter{} - err := c.initConverterFile(npmCacheFile) + c := &Converter{ + EnableV2NPM: true, + } + err := c.initConverterFile(npmCacheFileV2) if err != nil { t.Errorf("error during initilizing converter : %v", err) } @@ -114,206 +191,65 @@ func TestGetRulesFromChain(t *testing.T) { expected []*pb.RuleResponse } - iptableChainAllowed := &NPMIPtable.Chain{Rules: make([]*NPMIPtable.Rule, 0)} - iptableChainNotAllowed := &NPMIPtable.Chain{Rules: make([]*NPMIPtable.Rule, 0)} - - m0 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-2173871756", "dst"}}, - } // ns-testnamespace - NAMESPACE - m1 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-837532042", "dst"}}, - } // app:frontend - KEYVALUELABELOFPOD - m2 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-370790958", "dst"}}, - } // k1:v0:v1 - NESTEDLABELOFPOD - m3 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-530439631", "dst"}}, - } // all-namespaces - KEYLABELOFNAMESPACE - m4 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-3050895063", "dst,dst"}}, - } // namedport:serve-80 - NAMEDPORTS - m5 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-2537389870", "dst"}}, - } // k0 - KEYLABELOFPOD - m6 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-1217484542", "dst"}}, - } // ns-namespace:test0 - KEYVALUELABELOFNAMESPACE - - m7 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-2173871756", "dst"}}, - } // ns-testnamespace - NAMESPACE - m8 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-837532042", "dst"}}, - } // app:frontend - KEYVALUELABELOFPOD - m9 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-370790958", "dst"}}, - } // k1:v0:v1 - NESTEDLABELOFPOD - m10 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-530439631", "dst"}}, - } // all-namespaces - KEYLABELOFNAMESPACE - m11 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-3050895063", "dst,dst"}}, - } // namedport:serve-80 - NAMEDPORTS - m12 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-2537389870", "dst"}}, - } // k0 - KEYLABELOFPOD - m13 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-1217484542", "dst"}}, - } // ns-namespace:test0 - KEYVALUELABELOFNAMESPACE - - m14 := &NPMIPtable.Module{ - Verb: "tcp", - OptionValueMap: map[string][]string{"dport": {"8000"}}, - } - m15 := &NPMIPtable.Module{ - Verb: "udp", - OptionValueMap: map[string][]string{"sport": {"53"}}, - } - - s0 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NAMESPACE, - Name: "ns-testnamespace", - HashedSetName: "azure-npm-2173871756", - Included: true, - } - s1 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYVALUELABELOFPOD, - Name: "app:frontend", - HashedSetName: "azure-npm-837532042", - Included: true, - } - s2 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NESTEDLABELOFPOD, - Name: "k1:v0:v1", - HashedSetName: "azure-npm-370790958", - Included: true, - } - s3 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYLABELOFNAMESPACE, - Name: "all-namespaces", - HashedSetName: "azure-npm-530439631", - Included: true, - } - s4 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NAMEDPORTS, - Name: "namedport:serve-80", - HashedSetName: "azure-npm-3050895063", - Included: true, - } - s5 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYLABELOFPOD, - Name: "k0", - HashedSetName: "azure-npm-2537389870", - Included: true, - } - s6 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYVALUELABELOFNAMESPACE, - Name: "ns-namespace:test0", - HashedSetName: "azure-npm-1217484542", - Included: true, - } - - s7 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NAMESPACE, - Name: "ns-testnamespace", - HashedSetName: "azure-npm-2173871756", - } - s8 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYVALUELABELOFPOD, - Name: "app:frontend", - HashedSetName: "azure-npm-837532042", - } - s9 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NESTEDLABELOFPOD, - Name: "k1:v0:v1", - HashedSetName: "azure-npm-370790958", - } - s10 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYLABELOFNAMESPACE, - Name: "all-namespaces", - HashedSetName: "azure-npm-530439631", - } - s11 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NAMEDPORTS, - Name: "namedport:serve-80", - HashedSetName: "azure-npm-3050895063", - } - s12 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYLABELOFPOD, - Name: "k0", - HashedSetName: "azure-npm-2537389870", - } - s13 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYVALUELABELOFNAMESPACE, - Name: "ns-namespace:test0", - HashedSetName: "azure-npm-1217484542", - } - - modules := []*NPMIPtable.Module{m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14, m15} - dstList := []*pb.RuleResponse_SetInfo{s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13} - - r1 := &NPMIPtable.Rule{ - Protocol: "tcp", - Target: &NPMIPtable.Target{Name: "MARK", OptionValueMap: map[string][]string{"set-xmark": {"0x2000/0xffffffff"}}}, - Modules: modules, - } - r2 := &NPMIPtable.Rule{ - Protocol: "", - Target: &NPMIPtable.Target{Name: "DROP", OptionValueMap: map[string][]string{}}, - Modules: modules, + rawchain := &NPMIPtable.Chain{ + Name: "AZURE-NPM-EGRESS", + Rules: []*NPMIPtable.Rule{ + { + Protocol: "", + Target: &NPMIPtable.Target{ + Name: "AZURE-NPM-EGRESS-2697641196", + OptionValueMap: map[string][]string{}, + }, + Modules: []*NPMIPtable.Module{ + { + Verb: "set", + OptionValueMap: map[string][]string{"match-set": {"azure-npm-3922407721", "src"}}, + }, + { + Verb: "set", + OptionValueMap: map[string][]string{"match-set": {"azure-npm-2837910840", "src"}}, + }, + { + Verb: "comment", + OptionValueMap: map[string][]string{"comment": {"EGRESS-POLICY-y/base-FROM-podlabel-pod:a-AND-ns-y-IN-ns-y"}}, + }, + }, + }, + }, } - iptableChainAllowed.Rules = append(iptableChainAllowed.Rules, r1) - iptableChainAllowed.Name = "AZURE-NPM-INGRESS-PORT" - - iptableChainNotAllowed.Rules = append(iptableChainNotAllowed.Rules, r2) - iptableChainNotAllowed.Name = "AZURE-NPM-INGRESS-DROPS" - - expectedMarkRes := []*pb.RuleResponse{{ - Chain: "AZURE-NPM-INGRESS-PORT", - SrcList: []*pb.RuleResponse_SetInfo{}, - DstList: dstList, - Protocol: "tcp", - DPort: 8000, - SPort: 53, - Allowed: true, - Direction: pb.Direction_INGRESS, - UnsortedIpset: map[string]string{"azure-npm-3050895063": "dst,dst"}, - }} - - expectedDropRes := []*pb.RuleResponse{{ - Chain: "AZURE-NPM-INGRESS-DROPS", - SrcList: []*pb.RuleResponse_SetInfo{}, - DstList: dstList, - Protocol: "", - DPort: 8000, - SPort: 53, + expected := []*pb.RuleResponse{{ + Chain: "AZURE-NPM-EGRESS", + SrcList: []*pb.RuleResponse_SetInfo{ + { + Type: pb.SetType_KEYLABELOFPOD, + Name: "podlabel-pod:a", + HashedSetName: "azure-npm-3922407721", + Included: true, + }, + { + Type: pb.SetType_NAMESPACE, + Name: "ns-y", + HashedSetName: "azure-npm-2837910840", + Included: true, + }, + }, + DstList: []*pb.RuleResponse_SetInfo{}, Allowed: false, - Direction: pb.Direction_INGRESS, - UnsortedIpset: map[string]string{"azure-npm-3050895063": "dst,dst"}, + Direction: pb.Direction_EGRESS, + UnsortedIpset: map[string]string{}, + JumpTo: "AZURE-NPM-EGRESS-2697641196", + Comment: "[EGRESS-POLICY-y/base-FROM-podlabel-pod:a-AND-ns-y-IN-ns-y]", }} testCases := map[string]*test{ - "allowed rule": {input: iptableChainAllowed, expected: expectedMarkRes}, - "not allowed rule": {input: iptableChainNotAllowed, expected: expectedDropRes}, + "allowed rule": {input: rawchain, expected: expected}, } - c := &Converter{} - err := c.initConverterFile(npmCacheFile) + c := &Converter{ + EnableV2NPM: true, + } + err := c.initConverterFile(npmCacheFileV2) if err != nil { t.Errorf("error during initilizing converter : %v", err) } @@ -321,13 +257,11 @@ func TestGetRulesFromChain(t *testing.T) { for name, test := range testCases { test := test t.Run(name, func(t *testing.T) { - actuatlReponsesArr, err := c.getRulesFromChain(test.input) + actualResponseArr, err := c.getRulesFromChain(test.input) if err != nil { t.Errorf("error during get rules : %v", err) } - if !reflect.DeepEqual(test.expected, actuatlReponsesArr) { - t.Errorf("got '%+v', expected '%+v'", actuatlReponsesArr, test.expected) - } + require.Exactly(t, test.expected, actualResponseArr) }) } } @@ -335,152 +269,26 @@ func TestGetRulesFromChain(t *testing.T) { func TestGetModulesFromRule(t *testing.T) { m0 := &NPMIPtable.Module{ Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-2173871756", "dst"}}, - } // ns-testnamespace - NAMESPACE + OptionValueMap: map[string][]string{"match-set": {"azure-npm-2837910840", "dst"}}, + } // ns-y - NAMESPACE m1 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-837532042", "dst"}}, - } // app:frontend - KEYVALUELABELOFPOD - m2 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-370790958", "dst"}}, - } // k1:v0:v1 - NESTEDLABELOFPOD - m3 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-530439631", "dst"}}, - } // all-namespaces - KEYLABELOFNAMESPACE - m4 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-3050895063", "dst,dst"}}, - } // namedport:serve-80 - NAMEDPORTS - m5 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-2537389870", "dst"}}, - } // k0 - KEYLABELOFPOD - m6 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"match-set": {"azure-npm-1217484542", "dst"}}, - } // ns-namespace:test0 - KEYVALUELABELOFNAMESPACE - - m7 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-2173871756", "dst"}}, - } // ns-testnamespace - NAMESPACE - m8 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-837532042", "dst"}}, - } // app:frontend - KEYVALUELABELOFPOD - m9 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-370790958", "dst"}}, - } // k1:v0:v1 - NESTEDLABELOFPOD - m10 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-530439631", "dst"}}, - } // all-namespaces - KEYLABELOFNAMESPACE - m11 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-3050895063", "dst,dst"}}, - } // namedport:serve-80 - NAMEDPORTS - m12 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-2537389870", "dst"}}, - } // k0 - KEYLABELOFPOD - m13 := &NPMIPtable.Module{ - Verb: "set", - OptionValueMap: map[string][]string{"not-match-set": {"azure-npm-1217484542", "dst"}}, - } // ns-namespace:test0 - KEYVALUELABELOFNAMESPACE - - m14 := &NPMIPtable.Module{ Verb: "tcp", OptionValueMap: map[string][]string{"dport": {"8000"}}, } - m15 := &NPMIPtable.Module{ + m2 := &NPMIPtable.Module{ Verb: "udp", OptionValueMap: map[string][]string{"sport": {"53"}}, } s0 := &pb.RuleResponse_SetInfo{ Type: pb.SetType_NAMESPACE, - Name: "ns-testnamespace", - HashedSetName: "azure-npm-2173871756", - Included: true, - } - s1 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYVALUELABELOFPOD, - Name: "app:frontend", - HashedSetName: "azure-npm-837532042", - Included: true, - } - s2 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NESTEDLABELOFPOD, - Name: "k1:v0:v1", - HashedSetName: "azure-npm-370790958", - Included: true, - } - s3 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYLABELOFNAMESPACE, - Name: "all-namespaces", - HashedSetName: "azure-npm-530439631", - Included: true, - } - s4 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NAMEDPORTS, - Name: "namedport:serve-80", - HashedSetName: "azure-npm-3050895063", - Included: true, - } - s5 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYLABELOFPOD, - Name: "k0", - HashedSetName: "azure-npm-2537389870", - Included: true, - } - s6 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYVALUELABELOFNAMESPACE, - Name: "ns-namespace:test0", - HashedSetName: "azure-npm-1217484542", + Name: "ns-y", + HashedSetName: "azure-npm-2837910840", Included: true, } - s7 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NAMESPACE, - Name: "ns-testnamespace", - HashedSetName: "azure-npm-2173871756", - } - s8 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYVALUELABELOFPOD, - Name: "app:frontend", - HashedSetName: "azure-npm-837532042", - } - s9 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NESTEDLABELOFPOD, - Name: "k1:v0:v1", - HashedSetName: "azure-npm-370790958", - } - s10 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYLABELOFNAMESPACE, - Name: "all-namespaces", - HashedSetName: "azure-npm-530439631", - } - s11 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_NAMEDPORTS, - Name: "namedport:serve-80", - HashedSetName: "azure-npm-3050895063", - } - s12 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYLABELOFPOD, - Name: "k0", - HashedSetName: "azure-npm-2537389870", - } - s13 := &pb.RuleResponse_SetInfo{ - Type: pb.SetType_KEYVALUELABELOFNAMESPACE, - Name: "ns-namespace:test0", - HashedSetName: "azure-npm-1217484542", - } - - modules := []*NPMIPtable.Module{m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14, m15} - dstList := []*pb.RuleResponse_SetInfo{s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13} + modules := []*NPMIPtable.Module{m0, m1, m2} + dstList := []*pb.RuleResponse_SetInfo{s0} expectedRuleResponse := &pb.RuleResponse{ Chain: "TEST", @@ -491,7 +299,7 @@ func TestGetModulesFromRule(t *testing.T) { SPort: 53, Allowed: true, Direction: pb.Direction_INGRESS, - UnsortedIpset: map[string]string{"azure-npm-3050895063": "dst,dst"}, + UnsortedIpset: make(map[string]string), } actualRuleResponse := &pb.RuleResponse{ @@ -501,8 +309,10 @@ func TestGetModulesFromRule(t *testing.T) { Direction: pb.Direction_INGRESS, } - c := &Converter{} - err := c.initConverterFile(npmCacheFile) + c := &Converter{ + EnableV2NPM: true, + } + err := c.initConverterFile(npmCacheFileV2) if err != nil { t.Errorf("error during initilizing converter : %v", err) } @@ -512,7 +322,5 @@ func TestGetModulesFromRule(t *testing.T) { t.Errorf("error during getNPMIPtable.ModulesFromRule : %v", err) } - if !reflect.DeepEqual(expectedRuleResponse, actualRuleResponse) { - t.Errorf("got '%+v', expected '%+v'", actualRuleResponse, expectedRuleResponse) - } + require.Exactly(t, expectedRuleResponse, actualRuleResponse) } diff --git a/npm/pkg/dataplane/debug/trafficanalyzer.go b/npm/pkg/dataplane/debug/trafficanalyzer.go index 3f777dd69c..2462989e12 100644 --- a/npm/pkg/dataplane/debug/trafficanalyzer.go +++ b/npm/pkg/dataplane/debug/trafficanalyzer.go @@ -1,17 +1,25 @@ -package dataplane +package debug import ( "fmt" + "log" "net" + "sort" "strconv" "strings" - controllersv1 "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/v1" + npmconfig "github.com/Azure/azure-container-networking/npm/config" + common "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" "github.com/Azure/azure-container-networking/npm/pkg/dataplane/pb" "github.com/Azure/azure-container-networking/npm/util" "google.golang.org/protobuf/encoding/protojson" ) +type TupleAndRule struct { + Tuple *Tuple + Rule *pb.RuleResponse +} + // Tuple struct type Tuple struct { RuleType string `json:"ruleType"` @@ -23,52 +31,85 @@ type Tuple struct { Protocol string `json:"protocol"` } -// Input struct -type Input struct { - Content string - Type InputType -} +func PrettyPrintTuples(tuples []*TupleAndRule, srcList map[string]*pb.RuleResponse_SetInfo, dstList map[string]*pb.RuleResponse_SetInfo) { //nolint: gocritic + allowedrules := []*TupleAndRule{} + for _, tuple := range tuples { + if tuple.Tuple.RuleType == "ALLOWED" { + allowedrules = append(allowedrules, tuple) + continue + } + /*tuple.Tuple.Direction == "EGRESS" { + fmt.Printf("\tProtocol: %s, Port: %s\n, Chain: %v", tuple.Tuple.Protocol, tuple.Tuple.SrcPort, tuple.Rule.Chain) + }*/ + } -// InputType indicates allowed typle for source and destination input -type InputType int32 + sort.Slice(allowedrules, func(i, j int) bool { + return allowedrules[i].Tuple.Direction == "EGRESS" + }) -const ( - // IPADDRS indicates the IP Address input type - IPADDRS InputType = 0 - // PODNAME indicates the podname input type - PODNAME InputType = 1 - // EXTERNAL indicates the external input type - EXTERNAL InputType = 2 -) + tuplechains := make(map[Tuple]string) + + fmt.Printf("Allowed:\n") + section := "" + for _, tuple := range allowedrules { + + if tuple.Tuple.Direction != section { + fmt.Printf("\t%s:\n", tuple.Tuple.Direction) + section = tuple.Tuple.Direction + } + + t := *tuple + if chain, ok := tuplechains[*t.Tuple]; ok { + // doesn't exist in map + if chain != t.Rule.Chain { + // we've seen this tuple before with a different chain, need to print + fmt.Printf("\t\tProtocol: %s, Port: %s, Chain: %v, Comment: %v\n", tuple.Tuple.Protocol, tuple.Tuple.DstPort, tuple.Rule.Chain, tuple.Rule.Comment) + } + } else { + // we haven't seen this tuple before, print everything + tuplechains[*t.Tuple] = t.Rule.Chain + fmt.Printf("\t\tProtocol: %s, Port: %s, Chain: %v, Comment: %v\n", tuple.Tuple.Protocol, tuple.Tuple.DstPort, tuple.Rule.Chain, tuple.Rule.Comment) + + } -var ipPodMap = make(map[string]*controllersv1.NpmPod) + } + fmt.Printf("Key:\n") + fmt.Printf("IPSets:") + fmt.Printf("\tSource IPSets:\n") + for i := range srcList { + fmt.Printf("\t\tName: %s, HashedName: %s,\n", srcList[i].Name, srcList[i].HashedSetName) + } + fmt.Printf("\tDestination IPSets:\n") + for i := range dstList { + fmt.Printf("\t\tName: %s, HashedName: %s,\n", dstList[i].Name, dstList[i].HashedSetName) + } +} // GetNetworkTuple read from node's NPM cache and iptables-save and // returns a list of hit rules between the source and the destination in // JSON format and a list of tuples from those rules. -func GetNetworkTuple(src, dst *Input) ([][]byte, []*Tuple, error) { - c := &Converter{} - +func (c *Converter) GetNetworkTuple(src, dst *common.Input, config *npmconfig.Config) ([][]byte, []*TupleAndRule, map[string]*pb.RuleResponse_SetInfo, map[string]*pb.RuleResponse_SetInfo, error) { //nolint: gocritic,lll allRules, err := c.GetProtobufRulesFromIptable("filter") if err != nil { - return nil, nil, fmt.Errorf("error occurred during get network tuple : %w", err) + return nil, nil, nil, nil, fmt.Errorf("error occurred during get network tuple : %w", err) } + + // after we have all rules from the AZURE-NPM chains in the filter table, get the network tuples of src and dst + return getNetworkTupleCommon(src, dst, c.NPMCache, allRules) } // GetNetworkTupleFile read from NPM cache and iptables-save files and // returns a list of hit rules between the source and the destination in // JSON format and a list of tuples from those rules. -func GetNetworkTupleFile( - src, dst *Input, +func (c *Converter) GetNetworkTupleFile( //nolint:gocritic + src, dst *common.Input, npmCacheFile string, iptableSaveFile string, -) ([][]byte, []*Tuple, error) { - - c := &Converter{} +) ([][]byte, []*TupleAndRule, map[string]*pb.RuleResponse_SetInfo, map[string]*pb.RuleResponse_SetInfo, error) { allRules, err := c.GetProtobufRulesFromIptableFile(util.IptablesFilterTable, npmCacheFile, iptableSaveFile) if err != nil { - return nil, nil, fmt.Errorf("error occurred during get network tuple : %w", err) + return nil, nil, nil, nil, fmt.Errorf("error occurred during get network tuple : %w", err) } return getNetworkTupleCommon(src, dst, c.NPMCache, allRules) @@ -76,28 +117,25 @@ func GetNetworkTupleFile( // Common function. func getNetworkTupleCommon( - src, dst *Input, - npmCache *controllersv1.Cache, - allRules []*pb.RuleResponse, -) ([][]byte, []*Tuple, error) { - - for _, pod := range npmCache.PodMap { - ipPodMap[pod.PodIP] = pod - } + src, dst *common.Input, + npmCache common.GenericCache, + allRules map[*pb.RuleResponse]struct{}, +) ([][]byte, []*TupleAndRule, map[string]*pb.RuleResponse_SetInfo, map[string]*pb.RuleResponse_SetInfo, error) { - srcPod, err := getNPMPod(src, npmCache) + srcPod, err := npmCache.GetPod(src) if err != nil { - return nil, nil, fmt.Errorf("error occurred during get source pod : %w", err) + return nil, nil, nil, nil, fmt.Errorf("error occurred during get source pod : %w", err) } - dstPod, err := getNPMPod(dst, npmCache) + dstPod, err := npmCache.GetPod(dst) if err != nil { - return nil, nil, fmt.Errorf("error occurred during get destination pod : %w", err) + return nil, nil, nil, nil, fmt.Errorf("error occurred during get destination pod : %w", err) } - hitRules, err := getHitRules(srcPod, dstPod, allRules, npmCache) + // find all rules where the source pod and dest pod exist + hitRules, srcSets, dstSets, err := getHitRules(srcPod, dstPod, allRules, npmCache) if err != nil { - return nil, nil, fmt.Errorf("%w", err) + return nil, nil, srcSets, dstSets, fmt.Errorf("%w", err) } ruleResListJSON := make([][]byte, 0) @@ -108,12 +146,12 @@ func getNetworkTupleCommon( for _, rule := range hitRules { ruleJSON, err := m.Marshal(rule) // pretty print if err != nil { - return nil, nil, fmt.Errorf("error occurred during marshalling : %w", err) + return nil, nil, srcSets, dstSets, fmt.Errorf("error occurred during marshalling : %w", err) } ruleResListJSON = append(ruleResListJSON, ruleJSON) } - resTupleList := make([]*Tuple, 0) + resTupleList := make([]*TupleAndRule, 0) for _, rule := range hitRules { tuple := generateTuple(srcPod, dstPod, rule) resTupleList = append(resTupleList, tuple) @@ -126,40 +164,10 @@ func getNetworkTupleCommon( // } // tupleResListJson = append(tupleResListJson, ruleJson) // } - return ruleResListJSON, resTupleList, nil -} - -func getNPMPod(input *Input, npmCache *controllersv1.Cache) (*controllersv1.NpmPod, error) { - switch input.Type { - case PODNAME: - if pod, ok := npmCache.PodMap[input.Content]; ok { - return pod, nil - } - return nil, errInvalidInput - case IPADDRS: - if pod, ok := ipPodMap[input.Content]; ok { - return pod, nil - } - return nil, errInvalidIPAddress - case EXTERNAL: - return &controllersv1.NpmPod{}, nil - default: - return nil, errInvalidInput - } -} - -// GetInputType returns the type of the input for GetNetworkTuple. -func GetInputType(input string) InputType { - if input == "External" { - return EXTERNAL - } else if ip := net.ParseIP(input); ip != nil { - return IPADDRS - } else { - return PODNAME - } + return ruleResListJSON, resTupleList, srcSets, dstSets, nil } -func generateTuple(src, dst *controllersv1.NpmPod, rule *pb.RuleResponse) *Tuple { +func generateTuple(src, dst *common.NpmPod, rule *pb.RuleResponse) *TupleAndRule { tuple := &Tuple{} if rule.Allowed { tuple.RuleType = "ALLOWED" @@ -180,7 +188,7 @@ func generateTuple(src, dst *controllersv1.NpmPod, rule *pb.RuleResponse) *Tuple if len(rule.SrcList) == 0 { tuple.SrcIP = ANY } else { - tuple.SrcIP = src.PodIP + tuple.SrcIP = src.IP() } if rule.SPort != 0 { tuple.SrcPort = strconv.Itoa(int(rule.SPort)) @@ -190,7 +198,7 @@ func generateTuple(src, dst *controllersv1.NpmPod, rule *pb.RuleResponse) *Tuple if len(rule.DstList) == 0 { tuple.DstIP = ANY } else { - tuple.DstIP = dst.PodIP + tuple.DstIP = dst.IP() } if rule.DPort != 0 { tuple.DstPort = strconv.Itoa(int(rule.DPort)) @@ -202,71 +210,88 @@ func generateTuple(src, dst *controllersv1.NpmPod, rule *pb.RuleResponse) *Tuple } else { tuple.Protocol = ANY } - return tuple + return &TupleAndRule{ + Tuple: tuple, + Rule: rule, + } } func getHitRules( - src, dst *controllersv1.NpmPod, - rules []*pb.RuleResponse, - npmCache *controllersv1.Cache, -) ([]*pb.RuleResponse, error) { + src, dst *common.NpmPod, + rules map[*pb.RuleResponse]struct{}, + npmCache common.GenericCache, +) ([]*pb.RuleResponse, map[string]*pb.RuleResponse_SetInfo, map[string]*pb.RuleResponse_SetInfo, error) { res := make([]*pb.RuleResponse, 0) - for _, rule := range rules { - matched := true + srcSets := make(map[string]*pb.RuleResponse_SetInfo, 0) + dstSets := make(map[string]*pb.RuleResponse_SetInfo, 0) + + for rule := range rules { + matchedSrc := false + matchedDst := false + // evalute all match set in src for _, setInfo := range rule.SrcList { - // evalute all match set in src if src.Namespace == "" { // internet - matched = false break } + matchedSource, err := evaluateSetInfo("src", setInfo, src, rule, npmCache) if err != nil { - return nil, fmt.Errorf("error occurred during evaluating source's set info : %w", err) + return nil, nil, nil, fmt.Errorf("error occurred during evaluating source's set info : %w", err) } - if !matchedSource { - matched = false + if matchedSource { + matchedSrc = true + srcSets[setInfo.HashedSetName] = setInfo break } } - if !matched { - continue - } + + // evaluate all match set in dst for _, setInfo := range rule.DstList { - // evaluate all match set in dst if dst.Namespace == "" { // internet - matched = false break } + matchedDestination, err := evaluateSetInfo("dst", setInfo, dst, rule, npmCache) if err != nil { - return nil, fmt.Errorf("error occurred during evaluating destination's set info : %w", err) + return nil, nil, nil, fmt.Errorf("error occurred during evaluating destination's set info : %w", err) } - if !matchedDestination { - matched = false + if matchedDestination { + + dstSets[setInfo.HashedSetName] = setInfo + matchedDst = true break } } - if matched { + + // conditions: + // add if src matches and there's no dst + // add if dst matches and there's no src + // add if src and dst match with both src and dst specified + + if (matchedSrc && len(rule.DstList) == 0) || + (matchedDst && len(rule.SrcList) == 0) || + (matchedSrc && matchedDst) { res = append(res, rule) } } + if len(res) == 0 { // either no hit rules or no rules at all. Both cases allow all traffic res = append(res, &pb.RuleResponse{Allowed: true}) } - return res, nil + return res, srcSets, dstSets, nil } // evalute an ipset to find out whether the pod's attributes match with the set func evaluateSetInfo( origin string, setInfo *pb.RuleResponse_SetInfo, - pod *controllersv1.NpmPod, + pod *common.NpmPod, rule *pb.RuleResponse, - npmCache *controllersv1.Cache, + npmCache common.GenericCache, ) (bool, error) { switch setInfo.Type { @@ -287,14 +312,14 @@ func evaluateSetInfo( case pb.SetType_CIDRBLOCKS: return matchCIDRBLOCKS(pod, setInfo), nil default: - return false, errSetType + return false, common.ErrSetType } } -func matchKEYVALUELABELOFNAMESPACE(pod *controllersv1.NpmPod, npmCache *controllersv1.Cache, setInfo *pb.RuleResponse_SetInfo) bool { +func matchKEYVALUELABELOFNAMESPACE(pod *common.NpmPod, npmCache common.GenericCache, setInfo *pb.RuleResponse_SetInfo) bool { srcNamespace := util.NamespacePrefix + pod.Namespace key, expectedValue := processKeyValueLabelOfNameSpace(setInfo.Name) - actualValue := npmCache.NsMap[srcNamespace].LabelsMap[key] + actualValue := npmCache.GetNamespaceLabel(srcNamespace, key) if expectedValue != actualValue { // if the value is required but does not match if setInfo.Included { @@ -305,10 +330,11 @@ func matchKEYVALUELABELOFNAMESPACE(pod *controllersv1.NpmPod, npmCache *controll return false } } + return true } -func matchNESTEDLABELOFPOD(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_SetInfo) bool { +func matchNESTEDLABELOFPOD(pod *common.NpmPod, setInfo *pb.RuleResponse_SetInfo) bool { // a function to split the key and the values and then combine the key with each value // return list of key value pairs which are keyvaluelabel of pod // one match then break @@ -327,13 +353,15 @@ func matchNESTEDLABELOFPOD(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_S if !hasOneKeyValuePair && setInfo.Included { return false } + return true } -func matchKEYLABELOFNAMESPACE(pod *controllersv1.NpmPod, npmCache *controllersv1.Cache, setInfo *pb.RuleResponse_SetInfo) bool { - srcNamespace := util.NamespacePrefix + pod.Namespace - key := strings.TrimPrefix(setInfo.Name, util.NamespacePrefix) - if _, ok := npmCache.NsMap[srcNamespace].LabelsMap[key]; ok { +func matchKEYLABELOFNAMESPACE(pod *common.NpmPod, npmCache common.GenericCache, setInfo *pb.RuleResponse_SetInfo) bool { + srcNamespace := pod.Namespace + key := strings.Split(strings.TrimPrefix(setInfo.Name, util.NamespaceLabelPrefix), ":") + included := npmCache.GetNamespaceLabel(srcNamespace, key[0]) + if included != "" && included == key[1] { return setInfo.Included } if setInfo.Included { @@ -343,7 +371,7 @@ func matchKEYLABELOFNAMESPACE(pod *controllersv1.NpmPod, npmCache *controllersv1 return true } -func matchNAMESPACE(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_SetInfo) bool { +func matchNAMESPACE(pod *common.NpmPod, setInfo *pb.RuleResponse_SetInfo) bool { srcNamespace := util.NamespacePrefix + pod.Namespace if setInfo.Name != srcNamespace || (setInfo.Name == srcNamespace && !setInfo.Included) { return false @@ -351,15 +379,16 @@ func matchNAMESPACE(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_SetInfo) return true } -func matchKEYVALUELABELOFPOD(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_SetInfo) bool { +func matchKEYVALUELABELOFPOD(pod *common.NpmPod, setInfo *pb.RuleResponse_SetInfo) bool { key, value := processKeyValueLabelOfPod(setInfo.Name) if pod.Labels[key] != value || (pod.Labels[key] == value && !setInfo.Included) { return false } + log.Printf("matched key value label of pod") return true } -func matchKEYLABELOFPOD(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_SetInfo) bool { +func matchKEYLABELOFPOD(pod *common.NpmPod, setInfo *pb.RuleResponse_SetInfo) bool { key := setInfo.Name if _, ok := pod.Labels[key]; ok { return setInfo.Included @@ -368,10 +397,11 @@ func matchKEYLABELOFPOD(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_SetI // if key does not exist but required in rule return false } + return true } -func matchNAMEDPORTS(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_SetInfo, rule *pb.RuleResponse, origin string) bool { +func matchNAMEDPORTS(pod *common.NpmPod, setInfo *pb.RuleResponse_SetInfo, rule *pb.RuleResponse, origin string) bool { portname := strings.TrimPrefix(setInfo.Name, util.NamedPortIPSetPrefix) for _, namedPort := range pod.ContainerPorts { if namedPort.Name == portname { @@ -389,13 +419,14 @@ func matchNAMEDPORTS(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_SetInfo } else { rule.DPort = namedPort.ContainerPort } + return true } } return false } -func matchCIDRBLOCKS(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_SetInfo) bool { +func matchCIDRBLOCKS(pod *common.NpmPod, setInfo *pb.RuleResponse_SetInfo) bool { matched := false for _, entry := range setInfo.Contents { entrySplitted := strings.Split(entry, " ") @@ -414,6 +445,7 @@ func matchCIDRBLOCKS(pod *controllersv1.NpmPod, setInfo *pb.RuleResponse_SetInfo } } } + return matched } @@ -429,7 +461,8 @@ func processKeyValueLabelOfPod(kv string) (string, string) { } func processNestedLabelOfPod(kv string) []string { - kvList := strings.Split(kv, ":") + str := strings.TrimPrefix(kv, util.NestedLabelPrefix) + kvList := strings.Split(str, ":") key := kvList[0] ret := make([]string, 0) for _, value := range kvList[1:] { diff --git a/npm/pkg/dataplane/debug/trafficanalyzer_test.go b/npm/pkg/dataplane/debug/trafficanalyzer_test.go index 8eed757005..e3ddaa41d4 100644 --- a/npm/pkg/dataplane/debug/trafficanalyzer_test.go +++ b/npm/pkg/dataplane/debug/trafficanalyzer_test.go @@ -1,11 +1,13 @@ -package dataplane +package debug import ( "crypto/sha256" "fmt" - "reflect" "sort" "testing" + + common "github.com/Azure/azure-container-networking/npm/pkg/controlplane/controllers/common" + "github.com/stretchr/testify/require" ) func AsSha256(o interface{}) string { @@ -28,17 +30,17 @@ func hashTheSortTupleList(tupleList []*Tuple) []string { func TestGetInputType(t *testing.T) { type testInput struct { input string - expected InputType + expected common.InputType } tests := map[string]*testInput{ - "external": {input: "External", expected: EXTERNAL}, - "podname": {input: "test/server", expected: PODNAME}, - "ipaddress": {input: "10.240.0.38", expected: IPADDRS}, + "external": {input: "External", expected: common.EXTERNAL}, + "podname": {input: "test/server", expected: common.NSPODNAME}, + "ipaddress": {input: "10.240.0.38", expected: common.IPADDRS}, } for name, test := range tests { test := test t.Run(name, func(t *testing.T) { - actualInputType := GetInputType(test.input) + actualInputType := common.GetInputType(test.input) if actualInputType != test.expected { t.Errorf("got '%+v', expected '%+v'", actualInputType, test.expected) } @@ -48,8 +50,8 @@ func TestGetInputType(t *testing.T) { func TestGetNetworkTuple(t *testing.T) { type srcDstPair struct { - src *Input - dst *Input + src *common.Input + dst *common.Input } type testInput struct { @@ -58,100 +60,24 @@ func TestGetNetworkTuple(t *testing.T) { } i0 := &srcDstPair{ - src: &Input{Content: "z/b", Type: PODNAME}, - dst: &Input{Content: "netpol-4537-x/a", Type: PODNAME}, - } - i1 := &srcDstPair{ - src: &Input{Content: "", Type: EXTERNAL}, - dst: &Input{Content: "testnamespace/a", Type: PODNAME}, - } - i2 := &srcDstPair{ - src: &Input{Content: "testnamespace/a", Type: PODNAME}, - dst: &Input{Content: "", Type: EXTERNAL}, - } - i3 := &srcDstPair{ - src: &Input{Content: "10.240.0.70", Type: IPADDRS}, - dst: &Input{Content: "10.240.0.13", Type: IPADDRS}, - } - i4 := &srcDstPair{ - src: &Input{Content: "", Type: EXTERNAL}, - dst: &Input{Content: "test/server", Type: PODNAME}, + src: &common.Input{Content: "y/b", Type: common.NSPODNAME}, + dst: &common.Input{Content: "x/b", Type: common.NSPODNAME}, } expected0 := []*Tuple{ - { - RuleType: "NOT ALLOWED", - Direction: "INGRESS", - SrcIP: "ANY", - SrcPort: "ANY", - DstIP: "10.240.0.13", - DstPort: "ANY", - Protocol: "ANY", - }, { RuleType: "ALLOWED", - Direction: "INGRESS", - SrcIP: "10.240.0.70", - SrcPort: "ANY", - DstIP: "10.240.0.13", - DstPort: "ANY", - Protocol: "ANY", - }, - { - RuleType: "ALLOWED", - Direction: "INGRESS", - SrcIP: "10.240.0.70", - SrcPort: "ANY", - DstIP: "10.240.0.13", - DstPort: "ANY", - Protocol: "ANY", - }, - } - - expected1 := []*Tuple{ - { - RuleType: "NOT ALLOWED", - Direction: "INGRESS", - SrcIP: "ANY", - SrcPort: "ANY", - DstIP: "10.240.0.12", - DstPort: "ANY", - Protocol: "ANY", - }, - { - RuleType: "NOT ALLOWED", - Direction: "INGRESS", - SrcIP: "ANY", - SrcPort: "ANY", - DstIP: "10.240.0.12", - DstPort: "ANY", - Protocol: "ANY", - }, - { - RuleType: "ALLOWED", - Direction: "INGRESS", - SrcIP: "ANY", - SrcPort: "ANY", - DstIP: "10.240.0.12", - DstPort: "ANY", - Protocol: "ANY", - }, - } - - expected2 := []*Tuple{ - { - RuleType: "NOT ALLOWED", Direction: "EGRESS", - SrcIP: "10.240.0.12", + SrcIP: "10.224.0.17", SrcPort: "ANY", - DstIP: "ANY", - DstPort: "ANY", - Protocol: "ANY", + DstIP: "10.224.0.20", + DstPort: "80", + Protocol: "tcp", }, { RuleType: "ALLOWED", Direction: "EGRESS", - SrcIP: "10.240.0.12", + SrcIP: "10.224.0.17", SrcPort: "ANY", DstIP: "ANY", DstPort: "53", @@ -160,89 +86,61 @@ func TestGetNetworkTuple(t *testing.T) { { RuleType: "ALLOWED", Direction: "EGRESS", - SrcIP: "10.240.0.12", + SrcIP: "10.224.0.17", SrcPort: "ANY", DstIP: "ANY", DstPort: "53", Protocol: "tcp", }, - } - - expected3 := []*Tuple{ - { - RuleType: "NOT ALLOWED", - Direction: "INGRESS", - SrcIP: "ANY", - SrcPort: "ANY", - DstIP: "10.240.0.13", - DstPort: "ANY", - Protocol: "ANY", - }, - { - RuleType: "ALLOWED", - Direction: "INGRESS", - SrcIP: "10.240.0.70", - SrcPort: "ANY", - DstIP: "10.240.0.13", - DstPort: "ANY", - Protocol: "ANY", - }, - { - RuleType: "ALLOWED", - Direction: "INGRESS", - SrcIP: "10.240.0.70", - SrcPort: "ANY", - DstIP: "10.240.0.13", - DstPort: "ANY", - Protocol: "ANY", - }, - } - expected4 := []*Tuple{ { RuleType: "ALLOWED", - Direction: "INGRESS", - SrcIP: "ANY", + Direction: "EGRESS", + SrcIP: "10.224.0.17", SrcPort: "ANY", - DstIP: "10.240.0.38", + DstIP: "10.224.0.20", DstPort: "80", Protocol: "tcp", }, { RuleType: "NOT ALLOWED", - Direction: "INGRESS", - SrcIP: "ANY", + Direction: "EGRESS", + SrcIP: "10.224.0.17", SrcPort: "ANY", - DstIP: "10.240.0.38", + DstIP: "ANY", DstPort: "ANY", Protocol: "ANY", }, } tests := map[string]*testInput{ - "podname to podname": {input: i0, expected: expected0}, - "internet to podname": {input: i1, expected: expected1}, - "podname to internet": {input: i2, expected: expected2}, - "ipaddress to ipaddress": {input: i3, expected: expected3}, - "namedport": {input: i4, expected: expected4}, + "podname to podname": {input: i0, expected: expected0}, } for name, test := range tests { test := test t.Run(name, func(t *testing.T) { sortedExpectedTupleList := hashTheSortTupleList(test.expected) - _, actualTupleList, err := GetNetworkTupleFile( + + c := &Converter{ + EnableV2NPM: true, + } + + _, actualTupleList, _, _, err := c.GetNetworkTupleFile( test.input.src, test.input.dst, - npmCacheFile, - iptableSaveFile, + npmCacheFileV2, + iptableSaveFileV2, ) - if err != nil { - t.Errorf("error during get network tuple : %v", err) - } - sortedActualTupleList := hashTheSortTupleList(actualTupleList) - if !reflect.DeepEqual(sortedExpectedTupleList, sortedActualTupleList) { - t.Errorf("got '%+v', expected '%+v'", sortedActualTupleList, sortedExpectedTupleList) + + require.NoError(t, err) + + tuplelist := []*Tuple{} + for i := range actualTupleList { + tuplelist = append(tuplelist, actualTupleList[i].Tuple) } + + sortedActualTupleList := hashTheSortTupleList(tuplelist) + require.Exactly(t, sortedExpectedTupleList, sortedActualTupleList) }) } } diff --git a/npm/pkg/dataplane/dpshim/dpshim.go b/npm/pkg/dataplane/dpshim/dpshim.go index afb97b57f1..cee8724775 100644 --- a/npm/pkg/dataplane/dpshim/dpshim.go +++ b/npm/pkg/dataplane/dpshim/dpshim.go @@ -452,7 +452,7 @@ func (dp *DPShim) ApplyDataPlane() error { return nil } -func (dp *DPShim) GetAllIPSets() []string { +func (dp *DPShim) GetAllIPSets() map[string]string { return nil } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager.go b/npm/pkg/dataplane/ipsets/ipsetmanager.go index a6a3f473ba..97bc170b54 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager.go @@ -38,7 +38,7 @@ type IPSetManager struct { setMap map[string]*IPSet dirtyCache dirtyCacheInterface ioShim *common.IOShim - sync.Mutex + sync.RWMutex } type IPSetManagerCfg struct { @@ -432,16 +432,14 @@ func (iMgr *IPSetManager) ApplyIPSets() error { return nil } -func (iMgr *IPSetManager) GetAllIPSets() []string { - iMgr.Lock() - defer iMgr.Unlock() - setNames := make([]string, len(iMgr.setMap)) - i := 0 - for setName := range iMgr.setMap { - setNames[i] = setName - i++ +func (iMgr *IPSetManager) GetAllIPSets() map[string]string { + iMgr.RLock() + defer iMgr.RUnlock() + setMap := make(map[string]string, len(iMgr.setMap)) + for _, metadata := range iMgr.setMap { + setMap[metadata.HashedName] = metadata.Name } - return setNames + return setMap } func (iMgr *IPSetManager) exists(name string) bool { diff --git a/npm/pkg/dataplane/mocks/genericdataplane_generated.go b/npm/pkg/dataplane/mocks/genericdataplane_generated.go index 7899453ba6..cf2ec3cbb7 100644 --- a/npm/pkg/dataplane/mocks/genericdataplane_generated.go +++ b/npm/pkg/dataplane/mocks/genericdataplane_generated.go @@ -2,7 +2,7 @@ // // Code generated by MockGen. DO NOT EDIT. -// Source: /home/vamsi/workspace/azure-container-networking/npm/pkg/dataplane/types.go +// Source: /home/matmerr/go/src/github.com/Azure/azure-container-networking/npm/pkg/dataplane/types.go // Package mocks is a generated GoMock package. package mocks @@ -135,10 +135,10 @@ func (mr *MockGenericDataplaneMockRecorder) DeleteIPSet(setMetadata, deleteOptio } // GetAllIPSets mocks base method. -func (m *MockGenericDataplane) GetAllIPSets() []string { +func (m *MockGenericDataplane) GetAllIPSets() map[string]string { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAllIPSets") - ret0, _ := ret[0].([]string) + ret0, _ := ret[0].(map[string]string) return ret0 } diff --git a/npm/pkg/dataplane/parse/parser.go b/npm/pkg/dataplane/parse/parser.go index e038f20763..1f7f747bbc 100644 --- a/npm/pkg/dataplane/parse/parser.go +++ b/npm/pkg/dataplane/parse/parser.go @@ -2,12 +2,19 @@ package parse import ( "bytes" + "errors" "fmt" "os" "os/exec" + "strings" + "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/npm/metrics" NPMIPtable "github.com/Azure/azure-container-networking/npm/pkg/dataplane/iptables" "github.com/Azure/azure-container-networking/npm/util" + npmerrors "github.com/Azure/azure-container-networking/npm/util/errors" + "k8s.io/klog" + utilexec "k8s.io/utils/exec" ) var ( @@ -19,12 +26,49 @@ var ( MinOptionLength = 2 ) +type IPTablesParser struct { + IOShim *common.IOShim +} + +// runCommand returns (stdout, stderr, error) +func (i *IPTablesParser) runCommand(command string, args ...string) ([]byte, error) { + klog.Infof("Executing iptables command %v %v", command, args) + + commandExec := i.IOShim.Exec.Command(command, args...) + output, err := commandExec.CombinedOutput() + + var exitError utilexec.ExitError + if ok := errors.As(err, &exitError); ok { + errCode := exitError.ExitStatus() + allArgsString := command + " " + strings.Join(args, " ") + msgStr := strings.TrimSuffix(string(output), "\n") + if errCode > 0 { + metrics.SendErrorLogAndMetric(util.IptmID, "error: There was an error running command: [%s %s] Stderr: [%v, %s]", util.Iptables, allArgsString, exitError, msgStr) + } + return output, npmerrors.SimpleErrorWrapper(fmt.Sprintf("failed to run iptables command [%s %s] Stderr: [%s]", util.Iptables, allArgsString, msgStr), exitError) + } + return output, nil +} + +// Iptables creates a Go object from specified iptable by calling iptables-save within node. +func (i *IPTablesParser) Iptables(tableName string) (*NPMIPtable.Table, error) { + cmdArgs := []string{util.IptablesTableFlag, string(tableName)} + + output, err := i.runCommand(util.IptablesSave, cmdArgs...) + if err != nil { + return nil, err + } + + chains := parseIptablesChainObject(tableName, output) + return &NPMIPtable.Table{Name: tableName, Chains: chains}, nil +} + // Iptables creates a Go object from specified iptable by calling iptables-save within node. func Iptables(tableName string) (*NPMIPtable.Table, error) { iptableBuffer := bytes.NewBuffer(nil) // TODO: need to get iptable's lock cmdArgs := []string{util.IptablesTableFlag, string(tableName)} - cmd := exec.Command(util.IptablesSave, cmdArgs...) //nolint:gosec + cmd := exec.Command(util.IptablesSave, cmdArgs...) //nolint:gosec // client usage is filter table only cmd.Stdout = iptableBuffer stderrBuffer := bytes.NewBuffer(nil) @@ -42,7 +86,7 @@ func Iptables(tableName string) (*NPMIPtable.Table, error) { } // IptablesFile creates a Go object from specified iptable by reading from an iptables-save file. -func IptablesFile(tableName string, iptableSaveFile string) (*NPMIPtable.Table, error) { +func IptablesFile(tableName, iptableSaveFile string) (*NPMIPtable.Table, error) { iptableBuffer := bytes.NewBuffer(nil) byteArray, err := os.ReadFile(iptableSaveFile) if err != nil { diff --git a/npm/pkg/dataplane/parse/parser_test.go b/npm/pkg/dataplane/parse/parser_test.go index bd526ea0a7..a5d880e644 100644 --- a/npm/pkg/dataplane/parse/parser_test.go +++ b/npm/pkg/dataplane/parse/parser_test.go @@ -2,12 +2,15 @@ package parse import ( "bytes" + "log" "reflect" "strings" "testing" + "github.com/Azure/azure-container-networking/common" NPMIPtable "github.com/Azure/azure-container-networking/npm/pkg/dataplane/iptables" "github.com/Azure/azure-container-networking/npm/util" + testutils "github.com/Azure/azure-container-networking/test/utils" ) func TestParseIptablesObjectFile(t *testing.T) { @@ -17,8 +20,24 @@ func TestParseIptablesObjectFile(t *testing.T) { } } +func TestParseIptablesObjectFileV2(t *testing.T) { + table, err := IptablesFile(util.IptablesFilterTable, "../testdata/iptablesave-v2") + if err != nil { + t.Fatal(err) + } + log.Printf("%v", table) +} + func TestParseIptablesObject(t *testing.T) { - _, err := Iptables(util.IptablesFilterTable) + calls := []testutils.TestCmd{ + {Cmd: []string{"iptables-save", "-t", "filter"}}, + } + + parser := IPTablesParser{ + IOShim: common.NewMockIOShim(calls), + } + + _, err := parser.Iptables(util.IptablesFilterTable) if err != nil { t.Fatal(err) } diff --git a/npm/pkg/dataplane/pb/rule.pb.go b/npm/pkg/dataplane/pb/rule.pb.go index e15818f9e2..dfd2e82fa0 100644 --- a/npm/pkg/dataplane/pb/rule.pb.go +++ b/npm/pkg/dataplane/pb/rule.pb.go @@ -37,6 +37,7 @@ const ( SetType_NAMEDPORTS SetType = 5 SetType_NESTEDLABELOFPOD SetType = 6 SetType_CIDRBLOCKS SetType = 7 + SetType_UNKNOWN SetType = 8 ) // Enum value maps for SetType. @@ -153,6 +154,8 @@ type RuleResponse struct { Allowed bool `protobuf:"varint,7,opt,name=Allowed,proto3" json:"Allowed,omitempty"` Direction Direction `protobuf:"varint,8,opt,name=Direction,proto3,enum=pb.Direction" json:"Direction,omitempty"` UnsortedIpset map[string]string `protobuf:"bytes,9,rep,name=UnsortedIpset,proto3" json:"UnsortedIpset,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + JumpTo string + Comment string } func (x *RuleResponse) Reset() { diff --git a/npm/pkg/dataplane/testdata/iptablesave-v1 b/npm/pkg/dataplane/testdata/iptablesave-v1 new file mode 100644 index 0000000000..7de9a44b8e --- /dev/null +++ b/npm/pkg/dataplane/testdata/iptablesave-v1 @@ -0,0 +1 @@ +# Generated by iptables-save v1.8.4 on Thu May 12 17:07:55 2022 *mangle :PREROUTING ACCEPT [108477033:45584700995] :INPUT ACCEPT [55477472:28982097300] :FORWARD ACCEPT [52999561:16602603695] :OUTPUT ACCEPT [71251265:28927626321] :POSTROUTING ACCEPT [124237346:45529421736] :KUBE-KUBELET-CANARY - [0:0] :KUBE-PROXY-CANARY - [0:0] COMMIT # Completed on Thu May 12 17:07:55 2022 # Generated by iptables-save v1.8.4 on Thu May 12 17:07:55 2022 *nat :PREROUTING ACCEPT [1:60] :INPUT ACCEPT [1:60] :OUTPUT ACCEPT [6:360] :POSTROUTING ACCEPT [1:60] :IP-MASQ-AGENT - [0:0] :KUBE-KUBELET-CANARY - [0:0] :KUBE-MARK-DROP - [0:0] :KUBE-MARK-MASQ - [0:0] :KUBE-NODEPORTS - [0:0] :KUBE-POSTROUTING - [0:0] :KUBE-PROXY-CANARY - [0:0] :KUBE-SEP-2NIF6FXTRQ7OA6YC - [0:0] :KUBE-SEP-3NUV56XFKXZPQ7VE - [0:0] :KUBE-SEP-3SWOO5SU5VT2SDH5 - [0:0] :KUBE-SEP-3UJDMLKHGLQ234DK - [0:0] :KUBE-SEP-5QKUXKBOY4F32YV4 - [0:0] :KUBE-SEP-7AQJSFXY7KAIDAFV - [0:0] :KUBE-SEP-7PR35NERFXWP4SUU - [0:0] :KUBE-SEP-AAYQATG2HTTEIN5E - [0:0] :KUBE-SEP-AT2RBRUC2OWGQKGK - [0:0] :KUBE-SEP-B3LOFO5FS34BELSM - [0:0] :KUBE-SEP-BW5NCMF6XLQD5Z7C - [0:0] :KUBE-SEP-C7XXPRGNSSBSWYWG - [0:0] :KUBE-SEP-CDWEJECVN5E77SPS - [0:0] :KUBE-SEP-CVR74FKCZQHJP6XU - [0:0] :KUBE-SEP-D6USX3SHHNLDTLZ4 - [0:0] :KUBE-SEP-DDYYTNCNK2CBJOG6 - [0:0] :KUBE-SEP-DVITOMBVK5RFH5OV - [0:0] :KUBE-SEP-ETWZS3IZF4DQ2POO - [0:0] :KUBE-SEP-FHN6TMGU7I6RJ57I - [0:0] :KUBE-SEP-FQN7YZH4YBVK3JGL - [0:0] :KUBE-SEP-I7MXUHVNHNYPDCAQ - [0:0] :KUBE-SEP-IURBH4FAWBJC7BWV - [0:0] :KUBE-SEP-IUXBNKRWAAJVMMOE - [0:0] :KUBE-SEP-J735GZRJA56M7LWN - [0:0] :KUBE-SEP-JXMN5RTB6IEP2R3G - [0:0] :KUBE-SEP-KWZXIJAIZ5YW2263 - [0:0] :KUBE-SEP-LEBM7TWV6SZUR4QC - [0:0] :KUBE-SEP-OVRAG5AEN2NTOFGR - [0:0] :KUBE-SEP-PEOIUSAXTKPHXEM7 - [0:0] :KUBE-SEP-QOWJUALK4DMCSLXZ - [0:0] :KUBE-SEP-QSIIVWB2QVRHJDF4 - [0:0] :KUBE-SEP-QTAQH6PLL7NX5JIN - [0:0] :KUBE-SEP-RCA64DVQ2IHBO65J - [0:0] :KUBE-SEP-RSZ4G6NR3LZB4KKB - [0:0] :KUBE-SEP-SJY65PKVLHF4FGYY - [0:0] :KUBE-SEP-SWAQUOJ7VTVYZLFG - [0:0] :KUBE-SEP-U67RV7GEMEZTGOGC - [0:0] :KUBE-SEP-UYHKV73LWON2WKWM - [0:0] :KUBE-SEP-V23CB5QNTPNGFKTD - [0:0] :KUBE-SEP-VGJ4OXGG6Y2JYTPD - [0:0] :KUBE-SEP-W5SW4IQEZMTPSYNG - [0:0] :KUBE-SEP-WV2EQ2BPABJVR24O - [0:0] :KUBE-SEP-YIGLVYA6O7DQFF7F - [0:0] :KUBE-SEP-YJMPROKGBWQ5364M - [0:0] :KUBE-SEP-ZN2WC5GDKXU7ZF2D - [0:0] :KUBE-SERVICES - [0:0] :KUBE-SVC-24VNCHYOB2X52C2X - [0:0] :KUBE-SVC-2JOEBGVIBN7STKJK - [0:0] :KUBE-SVC-3S4YIWM7TJIPODNX - [0:0] :KUBE-SVC-5OWDM3VLLNBGM5TX - [0:0] :KUBE-SVC-6QUKZ244WBJIQPL2 - [0:0] :KUBE-SVC-A5Q7JOQ6EZ6U2DSF - [0:0] :KUBE-SVC-BE7PK7AJGKHTCEL4 - [0:0] :KUBE-SVC-BKDYZEC4L2UL4IMK - [0:0] :KUBE-SVC-BQ3MHMU4NCXF4EXK - [0:0] :KUBE-SVC-CCXIAI2X5XPGA6LH - [0:0] :KUBE-SVC-CZKTCEE7X2YRP4IS - [0:0] :KUBE-SVC-D3LXG2LS52OYEL3B - [0:0] :KUBE-SVC-D7EDA2TJHHONNNBM - [0:0] :KUBE-SVC-DJ4J2L6IPYZ4WQRG - [0:0] :KUBE-SVC-DURNRPMC3MU6HSAU - [0:0] :KUBE-SVC-ERIFXISQEP7F7OF4 - [0:0] :KUBE-SVC-FUWW5BMBVQHZ4M6Z - [0:0] :KUBE-SVC-G6T3TND64D5NPL53 - [0:0] :KUBE-SVC-HTSFCVQQUEHP7H27 - [0:0] :KUBE-SVC-I7YZ4BEHBIRBZRN3 - [0:0] :KUBE-SVC-IDYN5E57OGBZ447H - [0:0] :KUBE-SVC-JWKN6CSSHIYY3SV5 - [0:0] :KUBE-SVC-K4BK4CDIQENF3HWY - [0:0] :KUBE-SVC-KCRSKJ6PUNXZAQVF - [0:0] :KUBE-SVC-KGNLBJN4HLTFPI7Z - [0:0] :KUBE-SVC-KML3S6LDDYEIMJHK - [0:0] :KUBE-SVC-LES3KET6DL2IL3FT - [0:0] :KUBE-SVC-MJFOJ62KMLMXY7X4 - [0:0] :KUBE-SVC-NFY24UBGSSEJRHL2 - [0:0] :KUBE-SVC-NPX46M4PTMTKRN6Y - [0:0] :KUBE-SVC-PVTFO465EEPOVL4V - [0:0] :KUBE-SVC-QMWWTXBG7KFJQKLO - [0:0] :KUBE-SVC-RJEXVRHLMOGTZJUZ - [0:0] :KUBE-SVC-RVN6PM2P5UHRSSKN - [0:0] :KUBE-SVC-SVQKUFZZUX6I6QTN - [0:0] :KUBE-SVC-TCOU7JCQXEZGVUNU - [0:0] :KUBE-SVC-XDDJMSFC2RZBVTHX - [0:0] :KUBE-SVC-XHKKBLVGNCT7FW4G - [0:0] :KUBE-SVC-XRZMHNSK2RZ45E4Y - [0:0] :KUBE-SVC-YM56X76GZ3NWP77N - [0:0] :KUBE-SVC-Z3G5C2S26AICYVIK - [0:0] -A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES -A OUTPUT -m comment --comment "kubernetes service portals" -j KUBE-SERVICES -A POSTROUTING -m comment --comment "kubernetes postrouting rules" -j KUBE-POSTROUTING -A POSTROUTING -m comment --comment "ip-masq-agent: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom IP-MASQ-AGENT chain" -m addrtype ! --dst-type LOCAL -j IP-MASQ-AGENT -A IP-MASQ-AGENT -d 10.224.0.0/12 -m comment --comment "ip-masq-agent: local traffic is not subject to MASQUERADE" -j RETURN -A IP-MASQ-AGENT -d 10.224.0.0/16 -m comment --comment "ip-masq-agent: local traffic is not subject to MASQUERADE" -j RETURN -A IP-MASQ-AGENT -d 10.0.0.0/16 -m comment --comment "ip-masq-agent: local traffic is not subject to MASQUERADE" -j RETURN -A IP-MASQ-AGENT -m comment --comment "ip-masq-agent: outbound traffic is subject to MASQUERADE (must be last in chain)" -j MASQUERADE -A KUBE-MARK-DROP -j MARK --set-xmark 0x8000/0x8000 -A KUBE-MARK-MASQ -j MARK --set-xmark 0x4000/0x4000 -A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN -A KUBE-POSTROUTING -j MARK --set-xmark 0x4000/0x0 -A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --random-fully -A KUBE-SEP-2NIF6FXTRQ7OA6YC -s 10.224.0.68/32 -m comment --comment "z/s-z-a:service-port-tcp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-2NIF6FXTRQ7OA6YC -p tcp -m comment --comment "z/s-z-a:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.68:80 -A KUBE-SEP-3NUV56XFKXZPQ7VE -s 10.224.0.42/32 -m comment --comment "z/s-z-c:service-port-tcp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-3NUV56XFKXZPQ7VE -p tcp -m comment --comment "z/s-z-c:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.42:80 -A KUBE-SEP-3SWOO5SU5VT2SDH5 -s 10.224.0.40/32 -m comment --comment "x/s-x-c:service-port-tcp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-3SWOO5SU5VT2SDH5 -p tcp -m comment --comment "x/s-x-c:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.40:80 -A KUBE-SEP-3UJDMLKHGLQ234DK -s 10.224.0.20/32 -m comment --comment "x/s-x-b:service-port-udp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-3UJDMLKHGLQ234DK -p udp -m comment --comment "x/s-x-b:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.20:81 -A KUBE-SEP-5QKUXKBOY4F32YV4 -s 10.224.0.66/32 -m comment --comment "kube-system/npm-metrics-cluster-service" -j KUBE-MARK-MASQ -A KUBE-SEP-5QKUXKBOY4F32YV4 -p tcp -m comment --comment "kube-system/npm-metrics-cluster-service" -m tcp -j DNAT --to-destination 10.224.0.66:10091 -A KUBE-SEP-7AQJSFXY7KAIDAFV -s 10.224.0.4/32 -m comment --comment "kube-system/npm-metrics-cluster-service" -j KUBE-MARK-MASQ -A KUBE-SEP-7AQJSFXY7KAIDAFV -p tcp -m comment --comment "kube-system/npm-metrics-cluster-service" -m tcp -j DNAT --to-destination 10.224.0.4:10091 -A KUBE-SEP-7PR35NERFXWP4SUU -s 10.224.0.15/32 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ -A KUBE-SEP-7PR35NERFXWP4SUU -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.224.0.15:53 -A KUBE-SEP-AAYQATG2HTTEIN5E -s 10.224.0.80/32 -m comment --comment "kube-system/metrics-server" -j KUBE-MARK-MASQ -A KUBE-SEP-AAYQATG2HTTEIN5E -p tcp -m comment --comment "kube-system/metrics-server" -m tcp -j DNAT --to-destination 10.224.0.80:443 -A KUBE-SEP-AT2RBRUC2OWGQKGK -s 10.224.0.17/32 -m comment --comment "y/s-y-b:service-port-tcp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-AT2RBRUC2OWGQKGK -p tcp -m comment --comment "y/s-y-b:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.17:80 -A KUBE-SEP-B3LOFO5FS34BELSM -s 10.224.0.20/32 -m comment --comment "x/s-x-b:service-port-tcp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-B3LOFO5FS34BELSM -p tcp -m comment --comment "x/s-x-b:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.20:80 -A KUBE-SEP-BW5NCMF6XLQD5Z7C -s 10.224.0.40/32 -m comment --comment "x/s-x-c:service-port-udp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-BW5NCMF6XLQD5Z7C -p udp -m comment --comment "x/s-x-c:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.40:81 -A KUBE-SEP-C7XXPRGNSSBSWYWG -s 10.224.0.13/32 -m comment --comment "z/s-z-b:service-port-tcp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-C7XXPRGNSSBSWYWG -p tcp -m comment --comment "z/s-z-b:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.13:80 -A KUBE-SEP-CDWEJECVN5E77SPS -s 10.224.0.68/32 -m comment --comment "z/s-z-a:service-port-tcp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-CDWEJECVN5E77SPS -p tcp -m comment --comment "z/s-z-a:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.68:81 -A KUBE-SEP-CVR74FKCZQHJP6XU -s 10.224.0.43/32 -m comment --comment "y/s-y-c:service-port-tcp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-CVR74FKCZQHJP6XU -p tcp -m comment --comment "y/s-y-c:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.43:81 -A KUBE-SEP-D6USX3SHHNLDTLZ4 -s 10.224.0.42/32 -m comment --comment "z/s-z-c:service-port-tcp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-D6USX3SHHNLDTLZ4 -p tcp -m comment --comment "z/s-z-c:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.42:81 -A KUBE-SEP-DDYYTNCNK2CBJOG6 -s 10.224.0.43/32 -m comment --comment "y/s-y-c:service-port-tcp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-DDYYTNCNK2CBJOG6 -p tcp -m comment --comment "y/s-y-c:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.43:80 -A KUBE-SEP-DVITOMBVK5RFH5OV -s 10.224.0.17/32 -m comment --comment "y/s-y-b:service-port-udp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-DVITOMBVK5RFH5OV -p udp -m comment --comment "y/s-y-b:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.17:80 -A KUBE-SEP-ETWZS3IZF4DQ2POO -s 10.224.0.87/32 -m comment --comment "x/s-x-a:service-port-tcp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-ETWZS3IZF4DQ2POO -p tcp -m comment --comment "x/s-x-a:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.87:80 -A KUBE-SEP-FHN6TMGU7I6RJ57I -s 10.224.0.87/32 -m comment --comment "x/s-x-a:service-port-udp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-FHN6TMGU7I6RJ57I -p udp -m comment --comment "x/s-x-a:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.87:80 -A KUBE-SEP-FQN7YZH4YBVK3JGL -s 10.224.0.15/32 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ -A KUBE-SEP-FQN7YZH4YBVK3JGL -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.224.0.15:53 -A KUBE-SEP-I7MXUHVNHNYPDCAQ -s 10.224.0.42/32 -m comment --comment "z/s-z-c:service-port-udp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-I7MXUHVNHNYPDCAQ -p udp -m comment --comment "z/s-z-c:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.42:81 -A KUBE-SEP-IURBH4FAWBJC7BWV -s 10.224.0.20/32 -m comment --comment "x/s-x-b:service-port-tcp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-IURBH4FAWBJC7BWV -p tcp -m comment --comment "x/s-x-b:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.20:81 -A KUBE-SEP-IUXBNKRWAAJVMMOE -s 10.224.0.42/32 -m comment --comment "z/s-z-c:service-port-udp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-IUXBNKRWAAJVMMOE -p udp -m comment --comment "z/s-z-c:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.42:80 -A KUBE-SEP-J735GZRJA56M7LWN -s 10.224.0.87/32 -m comment --comment "x/s-x-a:service-port-tcp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-J735GZRJA56M7LWN -p tcp -m comment --comment "x/s-x-a:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.87:81 -A KUBE-SEP-JXMN5RTB6IEP2R3G -s 10.224.0.70/32 -m comment --comment "y/s-y-a:service-port-tcp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-JXMN5RTB6IEP2R3G -p tcp -m comment --comment "y/s-y-a:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.70:80 -A KUBE-SEP-KWZXIJAIZ5YW2263 -s 10.224.0.68/32 -m comment --comment "z/s-z-a:service-port-udp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-KWZXIJAIZ5YW2263 -p udp -m comment --comment "z/s-z-a:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.68:81 -A KUBE-SEP-LEBM7TWV6SZUR4QC -s 10.224.0.13/32 -m comment --comment "z/s-z-b:service-port-tcp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-LEBM7TWV6SZUR4QC -p tcp -m comment --comment "z/s-z-b:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.13:81 -A KUBE-SEP-OVRAG5AEN2NTOFGR -s 10.224.0.68/32 -m comment --comment "z/s-z-a:service-port-udp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-OVRAG5AEN2NTOFGR -p udp -m comment --comment "z/s-z-a:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.68:80 -A KUBE-SEP-PEOIUSAXTKPHXEM7 -s 10.224.0.17/32 -m comment --comment "y/s-y-b:service-port-tcp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-PEOIUSAXTKPHXEM7 -p tcp -m comment --comment "y/s-y-b:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.17:81 -A KUBE-SEP-QOWJUALK4DMCSLXZ -s 10.224.0.13/32 -m comment --comment "z/s-z-b:service-port-udp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-QOWJUALK4DMCSLXZ -p udp -m comment --comment "z/s-z-b:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.13:81 -A KUBE-SEP-QSIIVWB2QVRHJDF4 -s 10.224.0.17/32 -m comment --comment "y/s-y-b:service-port-udp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-QSIIVWB2QVRHJDF4 -p udp -m comment --comment "y/s-y-b:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.17:81 -A KUBE-SEP-QTAQH6PLL7NX5JIN -s 10.224.0.40/32 -m comment --comment "x/s-x-c:service-port-udp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-QTAQH6PLL7NX5JIN -p udp -m comment --comment "x/s-x-c:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.40:80 -A KUBE-SEP-RCA64DVQ2IHBO65J -s 10.224.0.43/32 -m comment --comment "y/s-y-c:service-port-udp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-RCA64DVQ2IHBO65J -p udp -m comment --comment "y/s-y-c:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.43:80 -A KUBE-SEP-RSZ4G6NR3LZB4KKB -s 40.64.79.12/32 -m comment --comment "default/kubernetes:https" -j KUBE-MARK-MASQ -A KUBE-SEP-RSZ4G6NR3LZB4KKB -p tcp -m comment --comment "default/kubernetes:https" -m tcp -j DNAT --to-destination 40.64.79.12:443 -A KUBE-SEP-SJY65PKVLHF4FGYY -s 10.224.0.41/32 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ -A KUBE-SEP-SJY65PKVLHF4FGYY -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.224.0.41:53 -A KUBE-SEP-SWAQUOJ7VTVYZLFG -s 10.224.0.13/32 -m comment --comment "z/s-z-b:service-port-udp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-SWAQUOJ7VTVYZLFG -p udp -m comment --comment "z/s-z-b:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.13:80 -A KUBE-SEP-U67RV7GEMEZTGOGC -s 10.224.0.70/32 -m comment --comment "y/s-y-a:service-port-tcp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-U67RV7GEMEZTGOGC -p tcp -m comment --comment "y/s-y-a:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.70:81 -A KUBE-SEP-UYHKV73LWON2WKWM -s 10.224.0.41/32 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ -A KUBE-SEP-UYHKV73LWON2WKWM -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.224.0.41:53 -A KUBE-SEP-V23CB5QNTPNGFKTD -s 10.224.0.40/32 -m comment --comment "x/s-x-c:service-port-tcp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-V23CB5QNTPNGFKTD -p tcp -m comment --comment "x/s-x-c:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.40:81 -A KUBE-SEP-VGJ4OXGG6Y2JYTPD -s 10.224.0.20/32 -m comment --comment "x/s-x-b:service-port-udp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-VGJ4OXGG6Y2JYTPD -p udp -m comment --comment "x/s-x-b:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.20:80 -A KUBE-SEP-W5SW4IQEZMTPSYNG -s 10.224.0.35/32 -m comment --comment "kube-system/npm-metrics-cluster-service" -j KUBE-MARK-MASQ -A KUBE-SEP-W5SW4IQEZMTPSYNG -p tcp -m comment --comment "kube-system/npm-metrics-cluster-service" -m tcp -j DNAT --to-destination 10.224.0.35:10091 -A KUBE-SEP-WV2EQ2BPABJVR24O -s 10.224.0.70/32 -m comment --comment "y/s-y-a:service-port-udp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-WV2EQ2BPABJVR24O -p udp -m comment --comment "y/s-y-a:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.70:81 -A KUBE-SEP-YIGLVYA6O7DQFF7F -s 10.224.0.87/32 -m comment --comment "x/s-x-a:service-port-udp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-YIGLVYA6O7DQFF7F -p udp -m comment --comment "x/s-x-a:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.87:81 -A KUBE-SEP-YJMPROKGBWQ5364M -s 10.224.0.70/32 -m comment --comment "y/s-y-a:service-port-udp-80" -j KUBE-MARK-MASQ -A KUBE-SEP-YJMPROKGBWQ5364M -p udp -m comment --comment "y/s-y-a:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.70:80 -A KUBE-SEP-ZN2WC5GDKXU7ZF2D -s 10.224.0.43/32 -m comment --comment "y/s-y-c:service-port-udp-81" -j KUBE-MARK-MASQ -A KUBE-SEP-ZN2WC5GDKXU7ZF2D -p udp -m comment --comment "y/s-y-c:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.43:81 -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.144.51/32 -p tcp -m comment --comment "z/s-z-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.144.51/32 -p tcp -m comment --comment "z/s-z-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-PVTFO465EEPOVL4V -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.164.41/32 -p udp -m comment --comment "x/s-x-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.164.41/32 -p udp -m comment --comment "x/s-x-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-5OWDM3VLLNBGM5TX -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.242.223/32 -p tcp -m comment --comment "y/s-y-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.242.223/32 -p tcp -m comment --comment "y/s-y-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-Z3G5C2S26AICYVIK -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.242.223/32 -p tcp -m comment --comment "y/s-y-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.242.223/32 -p tcp -m comment --comment "y/s-y-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-BKDYZEC4L2UL4IMK -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.144.51/32 -p udp -m comment --comment "z/s-z-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.144.51/32 -p udp -m comment --comment "z/s-z-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-I7YZ4BEHBIRBZRN3 -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.144.51/32 -p tcp -m comment --comment "z/s-z-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.144.51/32 -p tcp -m comment --comment "z/s-z-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-CZKTCEE7X2YRP4IS -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.87.211/32 -p tcp -m comment --comment "x/s-x-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.87.211/32 -p tcp -m comment --comment "x/s-x-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-LES3KET6DL2IL3FT -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport 53 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport 53 -j KUBE-SVC-ERIFXISQEP7F7OF4 -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.214.76/32 -p tcp -m comment --comment "y/s-y-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.214.76/32 -p tcp -m comment --comment "y/s-y-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-FUWW5BMBVQHZ4M6Z -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.242.223/32 -p udp -m comment --comment "y/s-y-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.242.223/32 -p udp -m comment --comment "y/s-y-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-KML3S6LDDYEIMJHK -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.46.211/32 -p tcp -m comment --comment "y/s-y-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.46.211/32 -p tcp -m comment --comment "y/s-y-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-HTSFCVQQUEHP7H27 -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.164.41/32 -p udp -m comment --comment "x/s-x-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.164.41/32 -p udp -m comment --comment "x/s-x-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-XDDJMSFC2RZBVTHX -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.169.38/32 -p udp -m comment --comment "z/s-z-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.169.38/32 -p udp -m comment --comment "z/s-z-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-YM56X76GZ3NWP77N -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.169.38/32 -p tcp -m comment --comment "z/s-z-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.169.38/32 -p tcp -m comment --comment "z/s-z-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-NFY24UBGSSEJRHL2 -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.87.211/32 -p udp -m comment --comment "x/s-x-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.87.211/32 -p udp -m comment --comment "x/s-x-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-RVN6PM2P5UHRSSKN -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.87.211/32 -p udp -m comment --comment "x/s-x-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.87.211/32 -p udp -m comment --comment "x/s-x-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-DURNRPMC3MU6HSAU -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.125.25/32 -p udp -m comment --comment "x/s-x-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.125.25/32 -p udp -m comment --comment "x/s-x-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-KGNLBJN4HLTFPI7Z -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.125.25/32 -p tcp -m comment --comment "x/s-x-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.125.25/32 -p tcp -m comment --comment "x/s-x-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-D3LXG2LS52OYEL3B -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.46.211/32 -p udp -m comment --comment "y/s-y-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.46.211/32 -p udp -m comment --comment "y/s-y-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-KCRSKJ6PUNXZAQVF -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.12.157/32 -p tcp -m comment --comment "z/s-z-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.12.157/32 -p tcp -m comment --comment "z/s-z-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-XRZMHNSK2RZ45E4Y -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.169.38/32 -p udp -m comment --comment "z/s-z-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.169.38/32 -p udp -m comment --comment "z/s-z-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-24VNCHYOB2X52C2X -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.164.41/32 -p tcp -m comment --comment "x/s-x-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.164.41/32 -p tcp -m comment --comment "x/s-x-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-XHKKBLVGNCT7FW4G -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.164.41/32 -p tcp -m comment --comment "x/s-x-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.164.41/32 -p tcp -m comment --comment "x/s-x-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-G6T3TND64D5NPL53 -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.87.211/32 -p tcp -m comment --comment "x/s-x-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.87.211/32 -p tcp -m comment --comment "x/s-x-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-BE7PK7AJGKHTCEL4 -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.0.10/32 -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport 53 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.0.10/32 -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport 53 -j KUBE-SVC-TCOU7JCQXEZGVUNU -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.52.70/32 -p tcp -m comment --comment "kube-system/metrics-server cluster IP" -m tcp --dport 443 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.52.70/32 -p tcp -m comment --comment "kube-system/metrics-server cluster IP" -m tcp --dport 443 -j KUBE-SVC-QMWWTXBG7KFJQKLO -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.125.25/32 -p udp -m comment --comment "x/s-x-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.125.25/32 -p udp -m comment --comment "x/s-x-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-D7EDA2TJHHONNNBM -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.12.157/32 -p udp -m comment --comment "z/s-z-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.12.157/32 -p udp -m comment --comment "z/s-z-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-MJFOJ62KMLMXY7X4 -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.214.76/32 -p udp -m comment --comment "y/s-y-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.214.76/32 -p udp -m comment --comment "y/s-y-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-6QUKZ244WBJIQPL2 -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.0.1/32 -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport 443 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.0.1/32 -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport 443 -j KUBE-SVC-NPX46M4PTMTKRN6Y -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.144.51/32 -p udp -m comment --comment "z/s-z-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.144.51/32 -p udp -m comment --comment "z/s-z-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-3S4YIWM7TJIPODNX -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.214.76/32 -p tcp -m comment --comment "y/s-y-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.214.76/32 -p tcp -m comment --comment "y/s-y-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-JWKN6CSSHIYY3SV5 -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.125.25/32 -p tcp -m comment --comment "x/s-x-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.125.25/32 -p tcp -m comment --comment "x/s-x-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-CCXIAI2X5XPGA6LH -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.214.76/32 -p udp -m comment --comment "y/s-y-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.214.76/32 -p udp -m comment --comment "y/s-y-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-RJEXVRHLMOGTZJUZ -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.242.223/32 -p udp -m comment --comment "y/s-y-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.242.223/32 -p udp -m comment --comment "y/s-y-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-IDYN5E57OGBZ447H -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.46.211/32 -p udp -m comment --comment "y/s-y-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.46.211/32 -p udp -m comment --comment "y/s-y-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-DJ4J2L6IPYZ4WQRG -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.82.246/32 -p tcp -m comment --comment "kube-system/npm-metrics-cluster-service cluster IP" -m tcp --dport 9000 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.82.246/32 -p tcp -m comment --comment "kube-system/npm-metrics-cluster-service cluster IP" -m tcp --dport 9000 -j KUBE-SVC-BQ3MHMU4NCXF4EXK -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.12.157/32 -p tcp -m comment --comment "z/s-z-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.12.157/32 -p tcp -m comment --comment "z/s-z-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-2JOEBGVIBN7STKJK -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.169.38/32 -p tcp -m comment --comment "z/s-z-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.169.38/32 -p tcp -m comment --comment "z/s-z-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-SVQKUFZZUX6I6QTN -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.46.211/32 -p tcp -m comment --comment "y/s-y-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.46.211/32 -p tcp -m comment --comment "y/s-y-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-A5Q7JOQ6EZ6U2DSF -A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.12.157/32 -p udp -m comment --comment "z/s-z-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ -A KUBE-SERVICES -d 10.0.12.157/32 -p udp -m comment --comment "z/s-z-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-K4BK4CDIQENF3HWY -A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS -A KUBE-SVC-24VNCHYOB2X52C2X -m comment --comment "z/s-z-c:service-port-udp-81" -j KUBE-SEP-I7MXUHVNHNYPDCAQ -A KUBE-SVC-2JOEBGVIBN7STKJK -m comment --comment "z/s-z-a:service-port-tcp-81" -j KUBE-SEP-CDWEJECVN5E77SPS -A KUBE-SVC-3S4YIWM7TJIPODNX -m comment --comment "z/s-z-b:service-port-udp-81" -j KUBE-SEP-QOWJUALK4DMCSLXZ -A KUBE-SVC-5OWDM3VLLNBGM5TX -m comment --comment "x/s-x-c:service-port-udp-80" -j KUBE-SEP-QTAQH6PLL7NX5JIN -A KUBE-SVC-6QUKZ244WBJIQPL2 -m comment --comment "y/s-y-b:service-port-udp-80" -j KUBE-SEP-DVITOMBVK5RFH5OV -A KUBE-SVC-A5Q7JOQ6EZ6U2DSF -m comment --comment "y/s-y-a:service-port-tcp-80" -j KUBE-SEP-JXMN5RTB6IEP2R3G -A KUBE-SVC-BE7PK7AJGKHTCEL4 -m comment --comment "x/s-x-b:service-port-tcp-80" -j KUBE-SEP-B3LOFO5FS34BELSM -A KUBE-SVC-BKDYZEC4L2UL4IMK -m comment --comment "y/s-y-c:service-port-tcp-81" -j KUBE-SEP-CVR74FKCZQHJP6XU -A KUBE-SVC-BQ3MHMU4NCXF4EXK -m comment --comment "kube-system/npm-metrics-cluster-service" -m statistic --mode random --probability 0.33333333349 -j KUBE-SEP-W5SW4IQEZMTPSYNG -A KUBE-SVC-BQ3MHMU4NCXF4EXK -m comment --comment "kube-system/npm-metrics-cluster-service" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-7AQJSFXY7KAIDAFV -A KUBE-SVC-BQ3MHMU4NCXF4EXK -m comment --comment "kube-system/npm-metrics-cluster-service" -j KUBE-SEP-5QKUXKBOY4F32YV4 -A KUBE-SVC-CCXIAI2X5XPGA6LH -m comment --comment "x/s-x-a:service-port-tcp-81" -j KUBE-SEP-J735GZRJA56M7LWN -A KUBE-SVC-CZKTCEE7X2YRP4IS -m comment --comment "z/s-z-b:service-port-tcp-81" -j KUBE-SEP-LEBM7TWV6SZUR4QC -A KUBE-SVC-D3LXG2LS52OYEL3B -m comment --comment "x/s-x-a:service-port-tcp-80" -j KUBE-SEP-ETWZS3IZF4DQ2POO -A KUBE-SVC-D7EDA2TJHHONNNBM -m comment --comment "x/s-x-a:service-port-udp-80" -j KUBE-SEP-FHN6TMGU7I6RJ57I -A KUBE-SVC-DJ4J2L6IPYZ4WQRG -m comment --comment "y/s-y-a:service-port-udp-80" -j KUBE-SEP-YJMPROKGBWQ5364M -A KUBE-SVC-DURNRPMC3MU6HSAU -m comment --comment "x/s-x-b:service-port-udp-80" -j KUBE-SEP-VGJ4OXGG6Y2JYTPD -A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-7PR35NERFXWP4SUU -A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-SEP-SJY65PKVLHF4FGYY -A KUBE-SVC-FUWW5BMBVQHZ4M6Z -m comment --comment "y/s-y-b:service-port-tcp-81" -j KUBE-SEP-PEOIUSAXTKPHXEM7 -A KUBE-SVC-G6T3TND64D5NPL53 -m comment --comment "x/s-x-c:service-port-tcp-80" -j KUBE-SEP-3SWOO5SU5VT2SDH5 -A KUBE-SVC-HTSFCVQQUEHP7H27 -m comment --comment "y/s-y-a:service-port-tcp-81" -j KUBE-SEP-U67RV7GEMEZTGOGC -A KUBE-SVC-I7YZ4BEHBIRBZRN3 -m comment --comment "z/s-z-b:service-port-udp-80" -j KUBE-SEP-SWAQUOJ7VTVYZLFG -A KUBE-SVC-IDYN5E57OGBZ447H -m comment --comment "y/s-y-c:service-port-udp-80" -j KUBE-SEP-RCA64DVQ2IHBO65J -A KUBE-SVC-JWKN6CSSHIYY3SV5 -m comment --comment "y/s-y-b:service-port-tcp-80" -j KUBE-SEP-AT2RBRUC2OWGQKGK -A KUBE-SVC-K4BK4CDIQENF3HWY -m comment --comment "z/s-z-a:service-port-udp-80" -j KUBE-SEP-OVRAG5AEN2NTOFGR -A KUBE-SVC-KCRSKJ6PUNXZAQVF -m comment --comment "y/s-y-a:service-port-udp-81" -j KUBE-SEP-WV2EQ2BPABJVR24O -A KUBE-SVC-KGNLBJN4HLTFPI7Z -m comment --comment "x/s-x-a:service-port-udp-81" -j KUBE-SEP-YIGLVYA6O7DQFF7F -A KUBE-SVC-KML3S6LDDYEIMJHK -m comment --comment "y/s-y-c:service-port-udp-81" -j KUBE-SEP-ZN2WC5GDKXU7ZF2D -A KUBE-SVC-LES3KET6DL2IL3FT -m comment --comment "x/s-x-b:service-port-tcp-81" -j KUBE-SEP-IURBH4FAWBJC7BWV -A KUBE-SVC-MJFOJ62KMLMXY7X4 -m comment --comment "z/s-z-a:service-port-udp-81" -j KUBE-SEP-KWZXIJAIZ5YW2263 -A KUBE-SVC-NFY24UBGSSEJRHL2 -m comment --comment "z/s-z-c:service-port-tcp-80" -j KUBE-SEP-3NUV56XFKXZPQ7VE -A KUBE-SVC-NPX46M4PTMTKRN6Y -m comment --comment "default/kubernetes:https" -j KUBE-SEP-RSZ4G6NR3LZB4KKB -A KUBE-SVC-PVTFO465EEPOVL4V -m comment --comment "z/s-z-b:service-port-tcp-80" -j KUBE-SEP-C7XXPRGNSSBSWYWG -A KUBE-SVC-QMWWTXBG7KFJQKLO -m comment --comment "kube-system/metrics-server" -j KUBE-SEP-AAYQATG2HTTEIN5E -A KUBE-SVC-RJEXVRHLMOGTZJUZ -m comment --comment "y/s-y-b:service-port-udp-81" -j KUBE-SEP-QSIIVWB2QVRHJDF4 -A KUBE-SVC-RVN6PM2P5UHRSSKN -m comment --comment "x/s-x-b:service-port-udp-81" -j KUBE-SEP-3UJDMLKHGLQ234DK -A KUBE-SVC-SVQKUFZZUX6I6QTN -m comment --comment "z/s-z-c:service-port-tcp-81" -j KUBE-SEP-D6USX3SHHNLDTLZ4 -A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-FQN7YZH4YBVK3JGL -A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns" -j KUBE-SEP-UYHKV73LWON2WKWM -A KUBE-SVC-XDDJMSFC2RZBVTHX -m comment --comment "x/s-x-c:service-port-udp-81" -j KUBE-SEP-BW5NCMF6XLQD5Z7C -A KUBE-SVC-XHKKBLVGNCT7FW4G -m comment --comment "x/s-x-c:service-port-tcp-81" -j KUBE-SEP-V23CB5QNTPNGFKTD -A KUBE-SVC-XRZMHNSK2RZ45E4Y -m comment --comment "z/s-z-a:service-port-tcp-80" -j KUBE-SEP-2NIF6FXTRQ7OA6YC -A KUBE-SVC-YM56X76GZ3NWP77N -m comment --comment "z/s-z-c:service-port-udp-80" -j KUBE-SEP-IUXBNKRWAAJVMMOE -A KUBE-SVC-Z3G5C2S26AICYVIK -m comment --comment "y/s-y-c:service-port-tcp-80" -j KUBE-SEP-DDYYTNCNK2CBJOG6 COMMIT # Completed on Thu May 12 17:07:55 2022 # Generated by iptables-save v1.8.4 on Thu May 12 17:07:55 2022 *security :INPUT ACCEPT [20259:6929114] :FORWARD ACCEPT [457:61848] :OUTPUT ACCEPT [19231:7019817] -A OUTPUT -d 168.63.129.16/32 -p tcp -m tcp --dport 53 -j ACCEPT -A OUTPUT -d 168.63.129.16/32 -p tcp -m owner --uid-owner 0 -j ACCEPT -A OUTPUT -d 168.63.129.16/32 -p tcp -m conntrack --ctstate INVALID,NEW -j DROP COMMIT # Completed on Thu May 12 17:07:55 2022 # Generated by iptables-save v1.8.4 on Thu May 12 17:07:55 2022 *filter :INPUT ACCEPT [5614:1636250] :FORWARD ACCEPT [0:0] :OUTPUT ACCEPT [7069:2514922] :AZURE-NPM - [0:0] :AZURE-NPM-ACCEPT - [0:0] :AZURE-NPM-EGRESS - [0:0] :AZURE-NPM-EGRESS-DROPS - [0:0] :AZURE-NPM-EGRESS-PORT - [0:0] :AZURE-NPM-EGRESS-TO - [0:0] :AZURE-NPM-INGRESS - [0:0] :AZURE-NPM-INGRESS-DROPS - [0:0] :AZURE-NPM-INGRESS-FROM - [0:0] :AZURE-NPM-INGRESS-PORT - [0:0] :KUBE-EXTERNAL-SERVICES - [0:0] :KUBE-FIREWALL - [0:0] :KUBE-FORWARD - [0:0] :KUBE-KUBELET-CANARY - [0:0] :KUBE-NODEPORTS - [0:0] :KUBE-PROXY-CANARY - [0:0] :KUBE-SERVICES - [0:0] -A INPUT -m comment --comment "kubernetes health check service ports" -j KUBE-NODEPORTS -A INPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES -A INPUT -j KUBE-FIREWALL -A FORWARD -m conntrack --ctstate NEW -j AZURE-NPM -A FORWARD -m comment --comment "kubernetes forwarding rules" -j KUBE-FORWARD -A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES -A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES -A FORWARD -d 168.63.129.16/32 -p tcp -m tcp --dport 80 -j DROP -A OUTPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES -A OUTPUT -j KUBE-FIREWALL -A AZURE-NPM -j AZURE-NPM-INGRESS -A AZURE-NPM -j AZURE-NPM-EGRESS -A AZURE-NPM -m mark --mark 0x3000 -m comment --comment ACCEPT-on-INGRESS-and-EGRESS-mark-0x3000 -j AZURE-NPM-ACCEPT -A AZURE-NPM -m mark --mark 0x2000 -m comment --comment ACCEPT-on-INGRESS-mark-0x2000 -j AZURE-NPM-ACCEPT -A AZURE-NPM -m mark --mark 0x1000 -m comment --comment ACCEPT-on-EGRESS-mark-0x1000 -j AZURE-NPM-ACCEPT -A AZURE-NPM -m state --state RELATED,ESTABLISHED -m comment --comment ACCEPT-on-connection-state -j ACCEPT -A AZURE-NPM-ACCEPT -m comment --comment Clear-AZURE-NPM-MARKS -j MARK --set-xmark 0x0/0xffffffff -A AZURE-NPM-ACCEPT -m comment --comment ACCEPT-All-packets -j ACCEPT -A AZURE-NPM-EGRESS -j AZURE-NPM-EGRESS-PORT -A AZURE-NPM-EGRESS -m mark --mark 0x3000 -m comment --comment RETURN-on-EGRESS-and-INGRESS-mark-0x3000 -j RETURN -A AZURE-NPM-EGRESS -m mark --mark 0x1000 -m comment --comment RETURN-on-EGRESS-mark-0x1000 -j RETURN -A AZURE-NPM-EGRESS -j AZURE-NPM-EGRESS-DROPS -A AZURE-NPM-EGRESS-DROPS -m mark --mark 0x3000 -m comment --comment RETURN-on-EGRESS-and-INGRESS-mark-0x3000 -j RETURN -A AZURE-NPM-EGRESS-DROPS -m mark --mark 0x1000 -m comment --comment RETURN-on-EGRESS-mark-0x1000 -j RETURN -A AZURE-NPM-EGRESS-DROPS -m set --match-set azure-npm-2837910840 src -m set --match-set azure-npm-3545492025 src -m comment --comment "DROP-ALL-FROM-pod:a-IN-ns-y" -j DROP -A AZURE-NPM-EGRESS-PORT -p tcp -m tcp --dport 53 -m set --match-set azure-npm-2837910840 src -m set --match-set azure-npm-3545492025 src -m comment --comment "ALLOW-ALL-TO-TCP-PORT-53-FROM-pod:a-IN-ns-y" -j MARK --set-xmark 0x1000/0x1000 -A AZURE-NPM-EGRESS-PORT -p udp -m udp --dport 53 -m set --match-set azure-npm-2837910840 src -m set --match-set azure-npm-3545492025 src -m comment --comment "ALLOW-ALL-TO-UDP-PORT-53-FROM-pod:a-IN-ns-y" -j MARK --set-xmark 0x1000/0x1000 -A AZURE-NPM-EGRESS-PORT -p tcp -m set --match-set azure-npm-2837910840 src -m set --match-set azure-npm-3545492025 src -m set --match-set azure-npm-718453262 dst -m set --match-set azure-npm-318152385 dst -m tcp --dport 80 -m comment --comment "ALLOW-pod:a-IN-ns-y-TO-ns-ns:z-AND-pod:a:b-AND-TCP-PORT-80" -j MARK --set-xmark 0x1000/0x1000 -A AZURE-NPM-EGRESS-PORT -p tcp -m set --match-set azure-npm-2837910840 src -m set --match-set azure-npm-3545492025 src -m set --match-set azure-npm-701675643 dst -m set --match-set azure-npm-318152385 dst -m tcp --dport 80 -m comment --comment "ALLOW-pod:a-IN-ns-y-TO-ns-ns:y-AND-pod:a:b-AND-TCP-PORT-80" -j MARK --set-xmark 0x1000/0x1000 -A AZURE-NPM-EGRESS-PORT -m set --match-set azure-npm-2064349730 src -m set --match-set azure-npm-826519261 src -m comment --comment "ALLOW-ALL-FROM-app:konnectivity-agent-IN-ns-kube-system" -j MARK --set-xmark 0x1000/0x1000 -A AZURE-NPM-EGRESS-PORT -m mark --mark 0x3000 -m comment --comment RETURN-on-EGRESS-and-INGRESS-mark-0x3000 -j RETURN -A AZURE-NPM-EGRESS-PORT -m mark --mark 0x1000 -m comment --comment RETURN-on-EGRESS-mark-0x1000 -j RETURN -A AZURE-NPM-EGRESS-PORT -m comment --comment ALL-JUMP-TO-AZURE-NPM-EGRESS-TO -j AZURE-NPM-EGRESS-TO -A AZURE-NPM-INGRESS -j AZURE-NPM-INGRESS-PORT -A AZURE-NPM-INGRESS -m mark --mark 0x2000 -m comment --comment RETURN-on-INGRESS-mark-0x2000 -j RETURN -A AZURE-NPM-INGRESS -j AZURE-NPM-INGRESS-DROPS -A AZURE-NPM-INGRESS-DROPS -m mark --mark 0x2000 -m comment --comment RETURN-on-INGRESS-mark-0x2000 -j RETURN -A AZURE-NPM-INGRESS-DROPS -m set --match-set azure-npm-2837910840 dst -m set --match-set azure-npm-3545492025 dst -m comment --comment "DROP-ALL-TO-pod:a-IN-ns-y" -j DROP -A AZURE-NPM-INGRESS-PORT -p tcp -m set --match-set azure-npm-701675643 src -m set --match-set azure-npm-3025643863 src -m set --match-set azure-npm-2837910840 dst -m set --match-set azure-npm-3545492025 dst -m tcp --dport 80 -m comment --comment "ALLOW-ns-ns:y-AND-pod:b:c-AND-TCP-PORT-80-TO-pod:a-IN-ns-y" -j MARK --set-xmark 0x2000/0xffffffff -A AZURE-NPM-INGRESS-PORT -p tcp -m set --match-set azure-npm-684898024 src -m set --match-set azure-npm-3025643863 src -m set --match-set azure-npm-2837910840 dst -m set --match-set azure-npm-3545492025 dst -m tcp --dport 80 -m comment --comment "ALLOW-ns-ns:x-AND-pod:b:c-AND-TCP-PORT-80-TO-pod:a-IN-ns-y" -j MARK --set-xmark 0x2000/0xffffffff -A AZURE-NPM-INGRESS-PORT -m mark --mark 0x2000 -m comment --comment RETURN-on-INGRESS-mark-0x2000 -j RETURN -A AZURE-NPM-INGRESS-PORT -m comment --comment ALL-JUMP-TO-AZURE-NPM-INGRESS-FROM -j AZURE-NPM-INGRESS-FROM -A KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP -A KUBE-FIREWALL ! -s 127.0.0.0/8 -d 127.0.0.0/8 -m comment --comment "block incoming localnet connections" -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP -A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP -A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT COMMIT # Completed on Thu May 12 17:07:55 2022 diff --git a/npm/pkg/dataplane/testdata/iptablesave-v2 b/npm/pkg/dataplane/testdata/iptablesave-v2 new file mode 100644 index 0000000000..a4e2768b73 --- /dev/null +++ b/npm/pkg/dataplane/testdata/iptablesave-v2 @@ -0,0 +1,411 @@ +# Generated by iptables-save v1.8.4 on Wed Apr 6 21:40:35 2022 +*mangle +:PREROUTING ACCEPT [1533177:1877010209] +:INPUT ACCEPT [1533177:1877010209] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [1570702:1685965745] +:POSTROUTING ACCEPT [1570702:1685965745] +:KUBE-KUBELET-CANARY - [0:0] +:KUBE-PROXY-CANARY - [0:0] +COMMIT +# Completed on Wed Apr 6 21:40:35 2022 +# Generated by iptables-save v1.8.4 on Wed Apr 6 21:40:35 2022 +*nat +:PREROUTING ACCEPT [1:60] +:INPUT ACCEPT [1:60] +:OUTPUT ACCEPT [24:1440] +:POSTROUTING ACCEPT [4:240] +:IP-MASQ-AGENT - [0:0] +:KUBE-KUBELET-CANARY - [0:0] +:KUBE-MARK-DROP - [0:0] +:KUBE-MARK-MASQ - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-POSTROUTING - [0:0] +:KUBE-PROXY-CANARY - [0:0] +:KUBE-SEP-2NIF6FXTRQ7OA6YC - [0:0] +:KUBE-SEP-3NUV56XFKXZPQ7VE - [0:0] +:KUBE-SEP-3SWOO5SU5VT2SDH5 - [0:0] +:KUBE-SEP-3UJDMLKHGLQ234DK - [0:0] +:KUBE-SEP-5QKUXKBOY4F32YV4 - [0:0] +:KUBE-SEP-5XQSQOS6AJCWJD6N - [0:0] +:KUBE-SEP-7AE45CCKWVVINGM7 - [0:0] +:KUBE-SEP-7AQJSFXY7KAIDAFV - [0:0] +:KUBE-SEP-AT2RBRUC2OWGQKGK - [0:0] +:KUBE-SEP-B3LOFO5FS34BELSM - [0:0] +:KUBE-SEP-BW5NCMF6XLQD5Z7C - [0:0] +:KUBE-SEP-C7XXPRGNSSBSWYWG - [0:0] +:KUBE-SEP-CDWEJECVN5E77SPS - [0:0] +:KUBE-SEP-CVR74FKCZQHJP6XU - [0:0] +:KUBE-SEP-D6USX3SHHNLDTLZ4 - [0:0] +:KUBE-SEP-DDYYTNCNK2CBJOG6 - [0:0] +:KUBE-SEP-DVITOMBVK5RFH5OV - [0:0] +:KUBE-SEP-ETWZS3IZF4DQ2POO - [0:0] +:KUBE-SEP-FHN6TMGU7I6RJ57I - [0:0] +:KUBE-SEP-FVX5CVUN3WB45Q4V - [0:0] +:KUBE-SEP-I7MXUHVNHNYPDCAQ - [0:0] +:KUBE-SEP-IURBH4FAWBJC7BWV - [0:0] +:KUBE-SEP-IUXBNKRWAAJVMMOE - [0:0] +:KUBE-SEP-J735GZRJA56M7LWN - [0:0] +:KUBE-SEP-JXMN5RTB6IEP2R3G - [0:0] +:KUBE-SEP-KWZXIJAIZ5YW2263 - [0:0] +:KUBE-SEP-LEBM7TWV6SZUR4QC - [0:0] +:KUBE-SEP-OVRAG5AEN2NTOFGR - [0:0] +:KUBE-SEP-PEOIUSAXTKPHXEM7 - [0:0] +:KUBE-SEP-PN3ZWU44T3MZSS7V - [0:0] +:KUBE-SEP-QOWJUALK4DMCSLXZ - [0:0] +:KUBE-SEP-QSIIVWB2QVRHJDF4 - [0:0] +:KUBE-SEP-QTAQH6PLL7NX5JIN - [0:0] +:KUBE-SEP-RCA64DVQ2IHBO65J - [0:0] +:KUBE-SEP-RSZ4G6NR3LZB4KKB - [0:0] +:KUBE-SEP-SWAQUOJ7VTVYZLFG - [0:0] +:KUBE-SEP-U67RV7GEMEZTGOGC - [0:0] +:KUBE-SEP-V23CB5QNTPNGFKTD - [0:0] +:KUBE-SEP-VGJ4OXGG6Y2JYTPD - [0:0] +:KUBE-SEP-W5SW4IQEZMTPSYNG - [0:0] +:KUBE-SEP-WV2EQ2BPABJVR24O - [0:0] +:KUBE-SEP-YIGLVYA6O7DQFF7F - [0:0] +:KUBE-SEP-YJMPROKGBWQ5364M - [0:0] +:KUBE-SEP-ZN2WC5GDKXU7ZF2D - [0:0] +:KUBE-SERVICES - [0:0] +:KUBE-SVC-24VNCHYOB2X52C2X - [0:0] +:KUBE-SVC-2JOEBGVIBN7STKJK - [0:0] +:KUBE-SVC-3S4YIWM7TJIPODNX - [0:0] +:KUBE-SVC-5OWDM3VLLNBGM5TX - [0:0] +:KUBE-SVC-6QUKZ244WBJIQPL2 - [0:0] +:KUBE-SVC-A5Q7JOQ6EZ6U2DSF - [0:0] +:KUBE-SVC-BE7PK7AJGKHTCEL4 - [0:0] +:KUBE-SVC-BKDYZEC4L2UL4IMK - [0:0] +:KUBE-SVC-BQ3MHMU4NCXF4EXK - [0:0] +:KUBE-SVC-CCXIAI2X5XPGA6LH - [0:0] +:KUBE-SVC-CZKTCEE7X2YRP4IS - [0:0] +:KUBE-SVC-D3LXG2LS52OYEL3B - [0:0] +:KUBE-SVC-D7EDA2TJHHONNNBM - [0:0] +:KUBE-SVC-DJ4J2L6IPYZ4WQRG - [0:0] +:KUBE-SVC-DURNRPMC3MU6HSAU - [0:0] +:KUBE-SVC-ERIFXISQEP7F7OF4 - [0:0] +:KUBE-SVC-FUWW5BMBVQHZ4M6Z - [0:0] +:KUBE-SVC-G6T3TND64D5NPL53 - [0:0] +:KUBE-SVC-HTSFCVQQUEHP7H27 - [0:0] +:KUBE-SVC-I7YZ4BEHBIRBZRN3 - [0:0] +:KUBE-SVC-IDYN5E57OGBZ447H - [0:0] +:KUBE-SVC-JWKN6CSSHIYY3SV5 - [0:0] +:KUBE-SVC-K4BK4CDIQENF3HWY - [0:0] +:KUBE-SVC-KCRSKJ6PUNXZAQVF - [0:0] +:KUBE-SVC-KGNLBJN4HLTFPI7Z - [0:0] +:KUBE-SVC-KML3S6LDDYEIMJHK - [0:0] +:KUBE-SVC-LES3KET6DL2IL3FT - [0:0] +:KUBE-SVC-MJFOJ62KMLMXY7X4 - [0:0] +:KUBE-SVC-NFY24UBGSSEJRHL2 - [0:0] +:KUBE-SVC-NPX46M4PTMTKRN6Y - [0:0] +:KUBE-SVC-PVTFO465EEPOVL4V - [0:0] +:KUBE-SVC-RJEXVRHLMOGTZJUZ - [0:0] +:KUBE-SVC-RVN6PM2P5UHRSSKN - [0:0] +:KUBE-SVC-SVQKUFZZUX6I6QTN - [0:0] +:KUBE-SVC-TCOU7JCQXEZGVUNU - [0:0] +:KUBE-SVC-XDDJMSFC2RZBVTHX - [0:0] +:KUBE-SVC-XHKKBLVGNCT7FW4G - [0:0] +:KUBE-SVC-XRZMHNSK2RZ45E4Y - [0:0] +:KUBE-SVC-YM56X76GZ3NWP77N - [0:0] +:KUBE-SVC-Z3G5C2S26AICYVIK - [0:0] +-A PREROUTING -m comment --comment "kubernetes service portals" -j KUBE-SERVICES +-A OUTPUT -m comment --comment "kubernetes service portals" -j KUBE-SERVICES +-A POSTROUTING -m comment --comment "kubernetes postrouting rules" -j KUBE-POSTROUTING +-A POSTROUTING -m comment --comment "ip-masq-agent: ensure nat POSTROUTING directs all non-LOCAL destination traffic to our custom IP-MASQ-AGENT chain" -m addrtype ! --dst-type LOCAL -j IP-MASQ-AGENT +-A IP-MASQ-AGENT -d 10.224.0.0/12 -m comment --comment "ip-masq-agent: local traffic is not subject to MASQUERADE" -j RETURN +-A IP-MASQ-AGENT -d 10.224.0.0/16 -m comment --comment "ip-masq-agent: local traffic is not subject to MASQUERADE" -j RETURN +-A IP-MASQ-AGENT -d 10.0.0.0/16 -m comment --comment "ip-masq-agent: local traffic is not subject to MASQUERADE" -j RETURN +-A IP-MASQ-AGENT -m comment --comment "ip-masq-agent: outbound traffic is subject to MASQUERADE (must be last in chain)" -j MASQUERADE +-A KUBE-MARK-DROP -j MARK --set-xmark 0x8000/0x8000 +-A KUBE-MARK-MASQ -j MARK --set-xmark 0x4000/0x4000 +-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN +-A KUBE-POSTROUTING -j MARK --set-xmark 0x4000/0x0 +-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --random-fully +-A KUBE-SEP-2NIF6FXTRQ7OA6YC -s 10.224.0.68/32 -m comment --comment "z/s-z-a:service-port-tcp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-2NIF6FXTRQ7OA6YC -p tcp -m comment --comment "z/s-z-a:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.68:80 +-A KUBE-SEP-3NUV56XFKXZPQ7VE -s 10.224.0.42/32 -m comment --comment "z/s-z-c:service-port-tcp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-3NUV56XFKXZPQ7VE -p tcp -m comment --comment "z/s-z-c:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.42:80 +-A KUBE-SEP-3SWOO5SU5VT2SDH5 -s 10.224.0.40/32 -m comment --comment "x/s-x-c:service-port-tcp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-3SWOO5SU5VT2SDH5 -p tcp -m comment --comment "x/s-x-c:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.40:80 +-A KUBE-SEP-3UJDMLKHGLQ234DK -s 10.224.0.20/32 -m comment --comment "x/s-x-b:service-port-udp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-3UJDMLKHGLQ234DK -p udp -m comment --comment "x/s-x-b:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.20:81 +-A KUBE-SEP-5QKUXKBOY4F32YV4 -s 10.224.0.66/32 -m comment --comment "kube-system/npm-metrics-cluster-service" -j KUBE-MARK-MASQ +-A KUBE-SEP-5QKUXKBOY4F32YV4 -p tcp -m comment --comment "kube-system/npm-metrics-cluster-service" -m tcp -j DNAT --to-destination 10.224.0.66:10091 +-A KUBE-SEP-5XQSQOS6AJCWJD6N -s 10.224.0.23/32 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ +-A KUBE-SEP-5XQSQOS6AJCWJD6N -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.224.0.23:53 +-A KUBE-SEP-7AE45CCKWVVINGM7 -s 10.224.0.69/32 -m comment --comment "kube-system/kube-dns:dns" -j KUBE-MARK-MASQ +-A KUBE-SEP-7AE45CCKWVVINGM7 -p udp -m comment --comment "kube-system/kube-dns:dns" -m udp -j DNAT --to-destination 10.224.0.69:53 +-A KUBE-SEP-7AQJSFXY7KAIDAFV -s 10.224.0.4/32 -m comment --comment "kube-system/npm-metrics-cluster-service" -j KUBE-MARK-MASQ +-A KUBE-SEP-7AQJSFXY7KAIDAFV -p tcp -m comment --comment "kube-system/npm-metrics-cluster-service" -m tcp -j DNAT --to-destination 10.224.0.4:10091 +-A KUBE-SEP-AT2RBRUC2OWGQKGK -s 10.224.0.17/32 -m comment --comment "y/s-y-b:service-port-tcp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-AT2RBRUC2OWGQKGK -p tcp -m comment --comment "y/s-y-b:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.17:80 +-A KUBE-SEP-B3LOFO5FS34BELSM -s 10.224.0.20/32 -m comment --comment "x/s-x-b:service-port-tcp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-B3LOFO5FS34BELSM -p tcp -m comment --comment "x/s-x-b:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.20:80 +-A KUBE-SEP-BW5NCMF6XLQD5Z7C -s 10.224.0.40/32 -m comment --comment "x/s-x-c:service-port-udp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-BW5NCMF6XLQD5Z7C -p udp -m comment --comment "x/s-x-c:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.40:81 +-A KUBE-SEP-C7XXPRGNSSBSWYWG -s 10.224.0.13/32 -m comment --comment "z/s-z-b:service-port-tcp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-C7XXPRGNSSBSWYWG -p tcp -m comment --comment "z/s-z-b:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.13:80 +-A KUBE-SEP-CDWEJECVN5E77SPS -s 10.224.0.68/32 -m comment --comment "z/s-z-a:service-port-tcp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-CDWEJECVN5E77SPS -p tcp -m comment --comment "z/s-z-a:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.68:81 +-A KUBE-SEP-CVR74FKCZQHJP6XU -s 10.224.0.43/32 -m comment --comment "y/s-y-c:service-port-tcp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-CVR74FKCZQHJP6XU -p tcp -m comment --comment "y/s-y-c:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.43:81 +-A KUBE-SEP-D6USX3SHHNLDTLZ4 -s 10.224.0.42/32 -m comment --comment "z/s-z-c:service-port-tcp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-D6USX3SHHNLDTLZ4 -p tcp -m comment --comment "z/s-z-c:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.42:81 +-A KUBE-SEP-DDYYTNCNK2CBJOG6 -s 10.224.0.43/32 -m comment --comment "y/s-y-c:service-port-tcp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-DDYYTNCNK2CBJOG6 -p tcp -m comment --comment "y/s-y-c:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.43:80 +-A KUBE-SEP-DVITOMBVK5RFH5OV -s 10.224.0.17/32 -m comment --comment "y/s-y-b:service-port-udp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-DVITOMBVK5RFH5OV -p udp -m comment --comment "y/s-y-b:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.17:80 +-A KUBE-SEP-ETWZS3IZF4DQ2POO -s 10.224.0.87/32 -m comment --comment "x/s-x-a:service-port-tcp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-ETWZS3IZF4DQ2POO -p tcp -m comment --comment "x/s-x-a:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.87:80 +-A KUBE-SEP-FHN6TMGU7I6RJ57I -s 10.224.0.87/32 -m comment --comment "x/s-x-a:service-port-udp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-FHN6TMGU7I6RJ57I -p udp -m comment --comment "x/s-x-a:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.87:80 +-A KUBE-SEP-FVX5CVUN3WB45Q4V -s 10.224.0.69/32 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ +-A KUBE-SEP-FVX5CVUN3WB45Q4V -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.224.0.69:53 +-A KUBE-SEP-I7MXUHVNHNYPDCAQ -s 10.224.0.42/32 -m comment --comment "z/s-z-c:service-port-udp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-I7MXUHVNHNYPDCAQ -p udp -m comment --comment "z/s-z-c:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.42:81 +-A KUBE-SEP-IURBH4FAWBJC7BWV -s 10.224.0.20/32 -m comment --comment "x/s-x-b:service-port-tcp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-IURBH4FAWBJC7BWV -p tcp -m comment --comment "x/s-x-b:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.20:81 +-A KUBE-SEP-IUXBNKRWAAJVMMOE -s 10.224.0.42/32 -m comment --comment "z/s-z-c:service-port-udp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-IUXBNKRWAAJVMMOE -p udp -m comment --comment "z/s-z-c:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.42:80 +-A KUBE-SEP-J735GZRJA56M7LWN -s 10.224.0.87/32 -m comment --comment "x/s-x-a:service-port-tcp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-J735GZRJA56M7LWN -p tcp -m comment --comment "x/s-x-a:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.87:81 +-A KUBE-SEP-JXMN5RTB6IEP2R3G -s 10.224.0.70/32 -m comment --comment "y/s-y-a:service-port-tcp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-JXMN5RTB6IEP2R3G -p tcp -m comment --comment "y/s-y-a:service-port-tcp-80" -m tcp -j DNAT --to-destination 10.224.0.70:80 +-A KUBE-SEP-KWZXIJAIZ5YW2263 -s 10.224.0.68/32 -m comment --comment "z/s-z-a:service-port-udp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-KWZXIJAIZ5YW2263 -p udp -m comment --comment "z/s-z-a:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.68:81 +-A KUBE-SEP-LEBM7TWV6SZUR4QC -s 10.224.0.13/32 -m comment --comment "z/s-z-b:service-port-tcp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-LEBM7TWV6SZUR4QC -p tcp -m comment --comment "z/s-z-b:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.13:81 +-A KUBE-SEP-OVRAG5AEN2NTOFGR -s 10.224.0.68/32 -m comment --comment "z/s-z-a:service-port-udp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-OVRAG5AEN2NTOFGR -p udp -m comment --comment "z/s-z-a:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.68:80 +-A KUBE-SEP-PEOIUSAXTKPHXEM7 -s 10.224.0.17/32 -m comment --comment "y/s-y-b:service-port-tcp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-PEOIUSAXTKPHXEM7 -p tcp -m comment --comment "y/s-y-b:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.17:81 +-A KUBE-SEP-PN3ZWU44T3MZSS7V -s 10.224.0.23/32 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-MARK-MASQ +-A KUBE-SEP-PN3ZWU44T3MZSS7V -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp" -m tcp -j DNAT --to-destination 10.224.0.23:53 +-A KUBE-SEP-QOWJUALK4DMCSLXZ -s 10.224.0.13/32 -m comment --comment "z/s-z-b:service-port-udp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-QOWJUALK4DMCSLXZ -p udp -m comment --comment "z/s-z-b:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.13:81 +-A KUBE-SEP-QSIIVWB2QVRHJDF4 -s 10.224.0.17/32 -m comment --comment "y/s-y-b:service-port-udp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-QSIIVWB2QVRHJDF4 -p udp -m comment --comment "y/s-y-b:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.17:81 +-A KUBE-SEP-QTAQH6PLL7NX5JIN -s 10.224.0.40/32 -m comment --comment "x/s-x-c:service-port-udp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-QTAQH6PLL7NX5JIN -p udp -m comment --comment "x/s-x-c:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.40:80 +-A KUBE-SEP-RCA64DVQ2IHBO65J -s 10.224.0.43/32 -m comment --comment "y/s-y-c:service-port-udp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-RCA64DVQ2IHBO65J -p udp -m comment --comment "y/s-y-c:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.43:80 +-A KUBE-SEP-RSZ4G6NR3LZB4KKB -s 40.64.79.12/32 -m comment --comment "default/kubernetes:https" -j KUBE-MARK-MASQ +-A KUBE-SEP-RSZ4G6NR3LZB4KKB -p tcp -m comment --comment "default/kubernetes:https" -m tcp -j DNAT --to-destination 40.64.79.12:443 +-A KUBE-SEP-SWAQUOJ7VTVYZLFG -s 10.224.0.13/32 -m comment --comment "z/s-z-b:service-port-udp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-SWAQUOJ7VTVYZLFG -p udp -m comment --comment "z/s-z-b:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.13:80 +-A KUBE-SEP-U67RV7GEMEZTGOGC -s 10.224.0.70/32 -m comment --comment "y/s-y-a:service-port-tcp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-U67RV7GEMEZTGOGC -p tcp -m comment --comment "y/s-y-a:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.70:81 +-A KUBE-SEP-V23CB5QNTPNGFKTD -s 10.224.0.40/32 -m comment --comment "x/s-x-c:service-port-tcp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-V23CB5QNTPNGFKTD -p tcp -m comment --comment "x/s-x-c:service-port-tcp-81" -m tcp -j DNAT --to-destination 10.224.0.40:81 +-A KUBE-SEP-VGJ4OXGG6Y2JYTPD -s 10.224.0.20/32 -m comment --comment "x/s-x-b:service-port-udp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-VGJ4OXGG6Y2JYTPD -p udp -m comment --comment "x/s-x-b:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.20:80 +-A KUBE-SEP-W5SW4IQEZMTPSYNG -s 10.224.0.35/32 -m comment --comment "kube-system/npm-metrics-cluster-service" -j KUBE-MARK-MASQ +-A KUBE-SEP-W5SW4IQEZMTPSYNG -p tcp -m comment --comment "kube-system/npm-metrics-cluster-service" -m tcp -j DNAT --to-destination 10.224.0.35:10091 +-A KUBE-SEP-WV2EQ2BPABJVR24O -s 10.224.0.70/32 -m comment --comment "y/s-y-a:service-port-udp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-WV2EQ2BPABJVR24O -p udp -m comment --comment "y/s-y-a:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.70:81 +-A KUBE-SEP-YIGLVYA6O7DQFF7F -s 10.224.0.87/32 -m comment --comment "x/s-x-a:service-port-udp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-YIGLVYA6O7DQFF7F -p udp -m comment --comment "x/s-x-a:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.87:81 +-A KUBE-SEP-YJMPROKGBWQ5364M -s 10.224.0.70/32 -m comment --comment "y/s-y-a:service-port-udp-80" -j KUBE-MARK-MASQ +-A KUBE-SEP-YJMPROKGBWQ5364M -p udp -m comment --comment "y/s-y-a:service-port-udp-80" -m udp -j DNAT --to-destination 10.224.0.70:80 +-A KUBE-SEP-ZN2WC5GDKXU7ZF2D -s 10.224.0.43/32 -m comment --comment "y/s-y-c:service-port-udp-81" -j KUBE-MARK-MASQ +-A KUBE-SEP-ZN2WC5GDKXU7ZF2D -p udp -m comment --comment "y/s-y-c:service-port-udp-81" -m udp -j DNAT --to-destination 10.224.0.43:81 +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.164.41/32 -p udp -m comment --comment "x/s-x-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.164.41/32 -p udp -m comment --comment "x/s-x-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-XDDJMSFC2RZBVTHX +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.82.246/32 -p tcp -m comment --comment "kube-system/npm-metrics-cluster-service cluster IP" -m tcp --dport 9000 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.82.246/32 -p tcp -m comment --comment "kube-system/npm-metrics-cluster-service cluster IP" -m tcp --dport 9000 -j KUBE-SVC-BQ3MHMU4NCXF4EXK +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.46.211/32 -p udp -m comment --comment "y/s-y-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.46.211/32 -p udp -m comment --comment "y/s-y-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-DJ4J2L6IPYZ4WQRG +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.12.157/32 -p udp -m comment --comment "z/s-z-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.12.157/32 -p udp -m comment --comment "z/s-z-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-MJFOJ62KMLMXY7X4 +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.144.51/32 -p udp -m comment --comment "z/s-z-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.144.51/32 -p udp -m comment --comment "z/s-z-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-I7YZ4BEHBIRBZRN3 +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.214.76/32 -p udp -m comment --comment "y/s-y-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.214.76/32 -p udp -m comment --comment "y/s-y-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-RJEXVRHLMOGTZJUZ +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.169.38/32 -p udp -m comment --comment "z/s-z-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.169.38/32 -p udp -m comment --comment "z/s-z-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-YM56X76GZ3NWP77N +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.0.10/32 -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport 53 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.0.10/32 -p udp -m comment --comment "kube-system/kube-dns:dns cluster IP" -m udp --dport 53 -j KUBE-SVC-TCOU7JCQXEZGVUNU +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.144.51/32 -p tcp -m comment --comment "z/s-z-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.144.51/32 -p tcp -m comment --comment "z/s-z-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-PVTFO465EEPOVL4V +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.242.223/32 -p tcp -m comment --comment "y/s-y-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.242.223/32 -p tcp -m comment --comment "y/s-y-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-BKDYZEC4L2UL4IMK +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.87.211/32 -p udp -m comment --comment "x/s-x-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.87.211/32 -p udp -m comment --comment "x/s-x-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-RVN6PM2P5UHRSSKN +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.46.211/32 -p udp -m comment --comment "y/s-y-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.46.211/32 -p udp -m comment --comment "y/s-y-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-KCRSKJ6PUNXZAQVF +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.125.25/32 -p tcp -m comment --comment "x/s-x-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.125.25/32 -p tcp -m comment --comment "x/s-x-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-CCXIAI2X5XPGA6LH +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport 53 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.0.10/32 -p tcp -m comment --comment "kube-system/kube-dns:dns-tcp cluster IP" -m tcp --dport 53 -j KUBE-SVC-ERIFXISQEP7F7OF4 +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.164.41/32 -p tcp -m comment --comment "x/s-x-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.164.41/32 -p tcp -m comment --comment "x/s-x-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-G6T3TND64D5NPL53 +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.12.157/32 -p tcp -m comment --comment "z/s-z-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.12.157/32 -p tcp -m comment --comment "z/s-z-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-XRZMHNSK2RZ45E4Y +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.87.211/32 -p tcp -m comment --comment "x/s-x-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.87.211/32 -p tcp -m comment --comment "x/s-x-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-BE7PK7AJGKHTCEL4 +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.46.211/32 -p tcp -m comment --comment "y/s-y-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.46.211/32 -p tcp -m comment --comment "y/s-y-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-A5Q7JOQ6EZ6U2DSF +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.125.25/32 -p tcp -m comment --comment "x/s-x-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.125.25/32 -p tcp -m comment --comment "x/s-x-a:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-D3LXG2LS52OYEL3B +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.169.38/32 -p tcp -m comment --comment "z/s-z-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.169.38/32 -p tcp -m comment --comment "z/s-z-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-NFY24UBGSSEJRHL2 +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.169.38/32 -p udp -m comment --comment "z/s-z-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.169.38/32 -p udp -m comment --comment "z/s-z-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-24VNCHYOB2X52C2X +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.12.157/32 -p tcp -m comment --comment "z/s-z-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.12.157/32 -p tcp -m comment --comment "z/s-z-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-2JOEBGVIBN7STKJK +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.87.211/32 -p udp -m comment --comment "x/s-x-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.87.211/32 -p udp -m comment --comment "x/s-x-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-DURNRPMC3MU6HSAU +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.87.211/32 -p tcp -m comment --comment "x/s-x-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.87.211/32 -p tcp -m comment --comment "x/s-x-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-LES3KET6DL2IL3FT +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.214.76/32 -p udp -m comment --comment "y/s-y-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.214.76/32 -p udp -m comment --comment "y/s-y-b:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-6QUKZ244WBJIQPL2 +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.0.1/32 -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport 443 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.0.1/32 -p tcp -m comment --comment "default/kubernetes:https cluster IP" -m tcp --dport 443 -j KUBE-SVC-NPX46M4PTMTKRN6Y +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.125.25/32 -p udp -m comment --comment "x/s-x-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.125.25/32 -p udp -m comment --comment "x/s-x-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-D7EDA2TJHHONNNBM +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.164.41/32 -p tcp -m comment --comment "x/s-x-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.164.41/32 -p tcp -m comment --comment "x/s-x-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-XHKKBLVGNCT7FW4G +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.164.41/32 -p udp -m comment --comment "x/s-x-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.164.41/32 -p udp -m comment --comment "x/s-x-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-5OWDM3VLLNBGM5TX +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.144.51/32 -p tcp -m comment --comment "z/s-z-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.144.51/32 -p tcp -m comment --comment "z/s-z-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-CZKTCEE7X2YRP4IS +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.144.51/32 -p udp -m comment --comment "z/s-z-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.144.51/32 -p udp -m comment --comment "z/s-z-b:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-3S4YIWM7TJIPODNX +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.242.223/32 -p tcp -m comment --comment "y/s-y-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.242.223/32 -p tcp -m comment --comment "y/s-y-c:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-Z3G5C2S26AICYVIK +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.46.211/32 -p tcp -m comment --comment "y/s-y-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.46.211/32 -p tcp -m comment --comment "y/s-y-a:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-HTSFCVQQUEHP7H27 +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.125.25/32 -p udp -m comment --comment "x/s-x-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.125.25/32 -p udp -m comment --comment "x/s-x-a:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-KGNLBJN4HLTFPI7Z +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.169.38/32 -p tcp -m comment --comment "z/s-z-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.169.38/32 -p tcp -m comment --comment "z/s-z-c:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-SVQKUFZZUX6I6QTN +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.12.157/32 -p udp -m comment --comment "z/s-z-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.12.157/32 -p udp -m comment --comment "z/s-z-a:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-K4BK4CDIQENF3HWY +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.242.223/32 -p udp -m comment --comment "y/s-y-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.242.223/32 -p udp -m comment --comment "y/s-y-c:service-port-udp-80 cluster IP" -m udp --dport 80 -j KUBE-SVC-IDYN5E57OGBZ447H +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.242.223/32 -p udp -m comment --comment "y/s-y-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.242.223/32 -p udp -m comment --comment "y/s-y-c:service-port-udp-81 cluster IP" -m udp --dport 81 -j KUBE-SVC-KML3S6LDDYEIMJHK +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.214.76/32 -p tcp -m comment --comment "y/s-y-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.214.76/32 -p tcp -m comment --comment "y/s-y-b:service-port-tcp-80 cluster IP" -m tcp --dport 80 -j KUBE-SVC-JWKN6CSSHIYY3SV5 +-A KUBE-SERVICES ! -s 10.224.0.0/16 -d 10.0.214.76/32 -p tcp -m comment --comment "y/s-y-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-MARK-MASQ +-A KUBE-SERVICES -d 10.0.214.76/32 -p tcp -m comment --comment "y/s-y-b:service-port-tcp-81 cluster IP" -m tcp --dport 81 -j KUBE-SVC-FUWW5BMBVQHZ4M6Z +-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS +-A KUBE-SVC-24VNCHYOB2X52C2X -m comment --comment "z/s-z-c:service-port-udp-81" -j KUBE-SEP-I7MXUHVNHNYPDCAQ +-A KUBE-SVC-2JOEBGVIBN7STKJK -m comment --comment "z/s-z-a:service-port-tcp-81" -j KUBE-SEP-CDWEJECVN5E77SPS +-A KUBE-SVC-3S4YIWM7TJIPODNX -m comment --comment "z/s-z-b:service-port-udp-81" -j KUBE-SEP-QOWJUALK4DMCSLXZ +-A KUBE-SVC-5OWDM3VLLNBGM5TX -m comment --comment "x/s-x-c:service-port-udp-80" -j KUBE-SEP-QTAQH6PLL7NX5JIN +-A KUBE-SVC-6QUKZ244WBJIQPL2 -m comment --comment "y/s-y-b:service-port-udp-80" -j KUBE-SEP-DVITOMBVK5RFH5OV +-A KUBE-SVC-A5Q7JOQ6EZ6U2DSF -m comment --comment "y/s-y-a:service-port-tcp-80" -j KUBE-SEP-JXMN5RTB6IEP2R3G +-A KUBE-SVC-BE7PK7AJGKHTCEL4 -m comment --comment "x/s-x-b:service-port-tcp-80" -j KUBE-SEP-B3LOFO5FS34BELSM +-A KUBE-SVC-BKDYZEC4L2UL4IMK -m comment --comment "y/s-y-c:service-port-tcp-81" -j KUBE-SEP-CVR74FKCZQHJP6XU +-A KUBE-SVC-BQ3MHMU4NCXF4EXK -m comment --comment "kube-system/npm-metrics-cluster-service" -m statistic --mode random --probability 0.33333333349 -j KUBE-SEP-W5SW4IQEZMTPSYNG +-A KUBE-SVC-BQ3MHMU4NCXF4EXK -m comment --comment "kube-system/npm-metrics-cluster-service" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-7AQJSFXY7KAIDAFV +-A KUBE-SVC-BQ3MHMU4NCXF4EXK -m comment --comment "kube-system/npm-metrics-cluster-service" -j KUBE-SEP-5QKUXKBOY4F32YV4 +-A KUBE-SVC-CCXIAI2X5XPGA6LH -m comment --comment "x/s-x-a:service-port-tcp-81" -j KUBE-SEP-J735GZRJA56M7LWN +-A KUBE-SVC-CZKTCEE7X2YRP4IS -m comment --comment "z/s-z-b:service-port-tcp-81" -j KUBE-SEP-LEBM7TWV6SZUR4QC +-A KUBE-SVC-D3LXG2LS52OYEL3B -m comment --comment "x/s-x-a:service-port-tcp-80" -j KUBE-SEP-ETWZS3IZF4DQ2POO +-A KUBE-SVC-D7EDA2TJHHONNNBM -m comment --comment "x/s-x-a:service-port-udp-80" -j KUBE-SEP-FHN6TMGU7I6RJ57I +-A KUBE-SVC-DJ4J2L6IPYZ4WQRG -m comment --comment "y/s-y-a:service-port-udp-80" -j KUBE-SEP-YJMPROKGBWQ5364M +-A KUBE-SVC-DURNRPMC3MU6HSAU -m comment --comment "x/s-x-b:service-port-udp-80" -j KUBE-SEP-VGJ4OXGG6Y2JYTPD +-A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-PN3ZWU44T3MZSS7V +-A KUBE-SVC-ERIFXISQEP7F7OF4 -m comment --comment "kube-system/kube-dns:dns-tcp" -j KUBE-SEP-FVX5CVUN3WB45Q4V +-A KUBE-SVC-FUWW5BMBVQHZ4M6Z -m comment --comment "y/s-y-b:service-port-tcp-81" -j KUBE-SEP-PEOIUSAXTKPHXEM7 +-A KUBE-SVC-G6T3TND64D5NPL53 -m comment --comment "x/s-x-c:service-port-tcp-80" -j KUBE-SEP-3SWOO5SU5VT2SDH5 +-A KUBE-SVC-HTSFCVQQUEHP7H27 -m comment --comment "y/s-y-a:service-port-tcp-81" -j KUBE-SEP-U67RV7GEMEZTGOGC +-A KUBE-SVC-I7YZ4BEHBIRBZRN3 -m comment --comment "z/s-z-b:service-port-udp-80" -j KUBE-SEP-SWAQUOJ7VTVYZLFG +-A KUBE-SVC-IDYN5E57OGBZ447H -m comment --comment "y/s-y-c:service-port-udp-80" -j KUBE-SEP-RCA64DVQ2IHBO65J +-A KUBE-SVC-JWKN6CSSHIYY3SV5 -m comment --comment "y/s-y-b:service-port-tcp-80" -j KUBE-SEP-AT2RBRUC2OWGQKGK +-A KUBE-SVC-K4BK4CDIQENF3HWY -m comment --comment "z/s-z-a:service-port-udp-80" -j KUBE-SEP-OVRAG5AEN2NTOFGR +-A KUBE-SVC-KCRSKJ6PUNXZAQVF -m comment --comment "y/s-y-a:service-port-udp-81" -j KUBE-SEP-WV2EQ2BPABJVR24O +-A KUBE-SVC-KGNLBJN4HLTFPI7Z -m comment --comment "x/s-x-a:service-port-udp-81" -j KUBE-SEP-YIGLVYA6O7DQFF7F +-A KUBE-SVC-KML3S6LDDYEIMJHK -m comment --comment "y/s-y-c:service-port-udp-81" -j KUBE-SEP-ZN2WC5GDKXU7ZF2D +-A KUBE-SVC-LES3KET6DL2IL3FT -m comment --comment "x/s-x-b:service-port-tcp-81" -j KUBE-SEP-IURBH4FAWBJC7BWV +-A KUBE-SVC-MJFOJ62KMLMXY7X4 -m comment --comment "z/s-z-a:service-port-udp-81" -j KUBE-SEP-KWZXIJAIZ5YW2263 +-A KUBE-SVC-NFY24UBGSSEJRHL2 -m comment --comment "z/s-z-c:service-port-tcp-80" -j KUBE-SEP-3NUV56XFKXZPQ7VE +-A KUBE-SVC-NPX46M4PTMTKRN6Y -m comment --comment "default/kubernetes:https" -j KUBE-SEP-RSZ4G6NR3LZB4KKB +-A KUBE-SVC-PVTFO465EEPOVL4V -m comment --comment "z/s-z-b:service-port-tcp-80" -j KUBE-SEP-C7XXPRGNSSBSWYWG +-A KUBE-SVC-RJEXVRHLMOGTZJUZ -m comment --comment "y/s-y-b:service-port-udp-81" -j KUBE-SEP-QSIIVWB2QVRHJDF4 +-A KUBE-SVC-RVN6PM2P5UHRSSKN -m comment --comment "x/s-x-b:service-port-udp-81" -j KUBE-SEP-3UJDMLKHGLQ234DK +-A KUBE-SVC-SVQKUFZZUX6I6QTN -m comment --comment "z/s-z-c:service-port-tcp-81" -j KUBE-SEP-D6USX3SHHNLDTLZ4 +-A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns" -m statistic --mode random --probability 0.50000000000 -j KUBE-SEP-5XQSQOS6AJCWJD6N +-A KUBE-SVC-TCOU7JCQXEZGVUNU -m comment --comment "kube-system/kube-dns:dns" -j KUBE-SEP-7AE45CCKWVVINGM7 +-A KUBE-SVC-XDDJMSFC2RZBVTHX -m comment --comment "x/s-x-c:service-port-udp-81" -j KUBE-SEP-BW5NCMF6XLQD5Z7C +-A KUBE-SVC-XHKKBLVGNCT7FW4G -m comment --comment "x/s-x-c:service-port-tcp-81" -j KUBE-SEP-V23CB5QNTPNGFKTD +-A KUBE-SVC-XRZMHNSK2RZ45E4Y -m comment --comment "z/s-z-a:service-port-tcp-80" -j KUBE-SEP-2NIF6FXTRQ7OA6YC +-A KUBE-SVC-YM56X76GZ3NWP77N -m comment --comment "z/s-z-c:service-port-udp-80" -j KUBE-SEP-IUXBNKRWAAJVMMOE +-A KUBE-SVC-Z3G5C2S26AICYVIK -m comment --comment "y/s-y-c:service-port-tcp-80" -j KUBE-SEP-DDYYTNCNK2CBJOG6 +COMMIT +# Completed on Wed Apr 6 21:40:35 2022 +# Generated by iptables-save v1.8.4 on Wed Apr 6 21:40:35 2022 +*filter +:INPUT ACCEPT [4360:1583973] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [5668:2831909] +:AZURE-NPM - [0:0] +:AZURE-NPM-ACCEPT - [0:0] +:AZURE-NPM-EGRESS - [0:0] +:AZURE-NPM-EGRESS-2697641196 - [0:0] +:AZURE-NPM-EGRESS-3618314628 - [0:0] +:AZURE-NPM-INGRESS - [0:0] +:AZURE-NPM-INGRESS-2697641196 - [0:0] +:AZURE-NPM-INGRESS-3750705395 - [0:0] +:AZURE-NPM-INGRESS-ALLOW-MARK - [0:0] +:KUBE-EXTERNAL-SERVICES - [0:0] +:KUBE-FIREWALL - [0:0] +:KUBE-FORWARD - [0:0] +:KUBE-KUBELET-CANARY - [0:0] +:KUBE-NODEPORTS - [0:0] +:KUBE-PROXY-CANARY - [0:0] +:KUBE-SERVICES - [0:0] +-A INPUT -m comment --comment "kubernetes health check service ports" -j KUBE-NODEPORTS +-A INPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES +-A INPUT -j KUBE-FIREWALL +-A FORWARD -m conntrack --ctstate NEW -j AZURE-NPM +-A FORWARD -m comment --comment "kubernetes forwarding rules" -j KUBE-FORWARD +-A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES +-A FORWARD -m conntrack --ctstate NEW -m comment --comment "kubernetes externally-visible service portals" -j KUBE-EXTERNAL-SERVICES +-A FORWARD -d 168.63.129.16/32 -p tcp -m tcp --dport 80 -j DROP +-A OUTPUT -m conntrack --ctstate NEW -m comment --comment "kubernetes service portals" -j KUBE-SERVICES +-A OUTPUT -j KUBE-FIREWALL +-A AZURE-NPM -j AZURE-NPM-INGRESS +-A AZURE-NPM -j AZURE-NPM-EGRESS +-A AZURE-NPM -j AZURE-NPM-ACCEPT +-A AZURE-NPM-ACCEPT -m comment --comment CLEAR-AZURE-NPM-MARKS -j MARK --set-xmark 0x0/0xffffffff +-A AZURE-NPM-ACCEPT -j ACCEPT +-A AZURE-NPM-EGRESS -m set --match-set azure-npm-3922407721 src -m set --match-set azure-npm-2837910840 src -m comment --comment "EGRESS-POLICY-y/base-FROM-podlabel-pod:a-AND-ns-y-IN-ns-y" -j AZURE-NPM-EGRESS-2697641196 +-A AZURE-NPM-EGRESS -m set --match-set azure-npm-4272224941 src -m set --match-set azure-npm-2064349730 src -m comment --comment "EGRESS-POLICY-kube-system/konnectivity-agent-FROM-podlabel-app:konnectivity-agent-AND-ns-kube-system-IN-ns-kube-system" -j AZURE-NPM-EGRESS-3618314628 +-A AZURE-NPM-EGRESS -m mark --mark 0x5000 -m comment --comment DROP-ON-EGRESS-DROP-MARK-0x5000 -j DROP +-A AZURE-NPM-EGRESS -m mark --mark 0x2000 -m comment --comment ACCEPT-ON-INGRESS-ALLOW-MARK-0x2000 -j AZURE-NPM-ACCEPT +-A AZURE-NPM-EGRESS-2697641196 -p tcp -m tcp --dport 80 -m set --match-set azure-npm-2146053937 dst -m set --match-set azure-npm-2682470511 dst -m comment --comment "ALLOW-TO-nslabel-ns:y-AND-nestedlabel-pod:a:b-ON-TCP-TO-PORT-80" -j AZURE-NPM-ACCEPT +-A AZURE-NPM-EGRESS-2697641196 -p tcp -m tcp --dport 80 -m set --match-set azure-npm-2095721080 dst -m set --match-set azure-npm-2682470511 dst -m comment --comment "ALLOW-TO-nslabel-ns:z-AND-nestedlabel-pod:a:b-ON-TCP-TO-PORT-80" -j AZURE-NPM-ACCEPT +-A AZURE-NPM-EGRESS-2697641196 -p udp -m udp --dport 53 -m comment --comment ALLOW-ALL-ON-UDP-TO-PORT-53 -j AZURE-NPM-ACCEPT +-A AZURE-NPM-EGRESS-2697641196 -p tcp -m tcp --dport 53 -m comment --comment ALLOW-ALL-ON-TCP-TO-PORT-53 -j AZURE-NPM-ACCEPT +-A AZURE-NPM-EGRESS-2697641196 -m comment --comment DROP-ALL -j MARK --set-xmark 0x5000/0xffffffff +-A AZURE-NPM-EGRESS-3618314628 -m comment --comment ALLOW-ALL -j AZURE-NPM-ACCEPT +-A AZURE-NPM-INGRESS -m set --match-set azure-npm-2064349730 dst -m comment --comment "INGRESS-POLICY-kube-system/default-deny-ingress-TO-ns-kube-system-IN-ns-kube-system" -j AZURE-NPM-INGRESS-3750705395 +-A AZURE-NPM-INGRESS -m set --match-set azure-npm-3922407721 dst -m set --match-set azure-npm-2837910840 dst -m comment --comment "INGRESS-POLICY-y/base-TO-podlabel-pod:a-AND-ns-y-IN-ns-y" -j AZURE-NPM-INGRESS-2697641196 +-A AZURE-NPM-INGRESS -m mark --mark 0x4000 -m comment --comment DROP-ON-INGRESS-DROP-MARK-0x4000 -j DROP +-A AZURE-NPM-INGRESS-2697641196 -p tcp -m tcp --dport 80 -m set --match-set azure-npm-2129276318 src -m set --match-set azure-npm-55798953 src -m comment --comment "ALLOW-FROM-nslabel-ns:x-AND-nestedlabel-pod:b:c-ON-TCP-TO-PORT-80" -j AZURE-NPM-INGRESS-ALLOW-MARK +-A AZURE-NPM-INGRESS-2697641196 -p tcp -m tcp --dport 80 -m set --match-set azure-npm-2146053937 src -m set --match-set azure-npm-55798953 src -m comment --comment "ALLOW-FROM-nslabel-ns:y-AND-nestedlabel-pod:b:c-ON-TCP-TO-PORT-80" -j AZURE-NPM-INGRESS-ALLOW-MARK +-A AZURE-NPM-INGRESS-2697641196 -m comment --comment DROP-ALL -j MARK --set-xmark 0x4000/0xffffffff +-A AZURE-NPM-INGRESS-3750705395 -m comment --comment DROP-ALL -j MARK --set-xmark 0x4000/0xffffffff +-A AZURE-NPM-INGRESS-ALLOW-MARK -m comment --comment SET-INGRESS-ALLOW-MARK-0x2000 -j MARK --set-xmark 0x2000/0xffffffff +-A AZURE-NPM-INGRESS-ALLOW-MARK -j AZURE-NPM-EGRESS +-A KUBE-FIREWALL -m comment --comment "kubernetes firewall for dropping marked packets" -m mark --mark 0x8000/0x8000 -j DROP +-A KUBE-FIREWALL ! -s 127.0.0.0/8 -d 127.0.0.0/8 -m comment --comment "block incoming localnet connections" -m conntrack ! --ctstate RELATED,ESTABLISHED,DNAT -j DROP +-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A KUBE-SERVICES -d 10.0.52.70/32 -p tcp -m comment --comment "kube-system/metrics-server has no endpoints" -m tcp --dport 443 -j REJECT --reject-with icmp-port-unreachable +COMMIT +# Completed on Wed Apr 6 21:40:35 2022 +# Generated by iptables-save v1.8.4 on Wed Apr 6 21:40:35 2022 +*security +:INPUT ACCEPT [4045:1449077] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [3826:1886289] +-A OUTPUT -d 168.63.129.16/32 -p tcp -m tcp --dport 53 -j ACCEPT +-A OUTPUT -d 168.63.129.16/32 -p tcp -m owner --uid-owner 0 -j ACCEPT +-A OUTPUT -d 168.63.129.16/32 -p tcp -m conntrack --ctstate INVALID,NEW -j DROP +COMMIT +# Completed on Wed Apr 6 21:40:35 2022 diff --git a/npm/pkg/dataplane/testdata/netpol.yaml b/npm/pkg/dataplane/testdata/netpol.yaml new file mode 100644 index 0000000000..bae2b955c8 --- /dev/null +++ b/npm/pkg/dataplane/testdata/netpol.yaml @@ -0,0 +1,82 @@ +# Expected combined: +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +# | TCP/80 | X/A | X/B | X/C | Y/A | Y/B | Y/C | Z/A | Z/B | Z/C | +# | TCP/81 | | | | | | | | | | +# | UDP/80 | | | | | | | | | | +# | UDP/81 | | | | | | | | | | +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +# | x/a | . . . . | . . . . | . . . . | X X X X | . . . . | . . . . | . . . . | . . . . | . . . . | +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +# | x/b | . . . . | . . . . | . . . . | . X X X | . . . . | . . . . | . . . . | . . . . | . . . . | +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +# | x/c | . . . . | . . . . | . . . . | . X X X | . . . . | . . . . | . . . . | . . . . | . . . . | +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +# | y/a | X X X X | X X X X | X X X X | X X X X | . X X X | X X X X | . X X X | . X X X | X X X X | +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +# | y/b | . . . . | . . . . | . . . . | . X X X | . . . . | . . . . | . . . . | . . . . | . . . . | +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +# | y/c | . . . . | . . . . | . . . . | . X X X | . . . . | . . . . | . . . . | . . . . | . . . . | +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +# | z/a | . . . . | . . . . | . . . . | X X X X | . . . . | . . . . | . . . . | . . . . | . . . . | +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +# | z/b | . . . . | . . . . | . . . . | X X X X | . . . . | . . . . | . . . . | . . . . | . . . . | +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +# | z/c | . . . . | . . . . | . . . . | X X X X | . . . . | . . . . | . . . . | . . . . | . . . . | +# +--------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ + +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + creationTimestamp: null + name: base + namespace: "y" +spec: + egress: + - ports: + - port: 80 + protocol: TCP + to: + - namespaceSelector: + matchExpressions: + - key: ns + operator: In + values: + - "y" + - z + podSelector: + matchExpressions: + - key: pod + operator: In + values: + - a + - b + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + ingress: + - from: + - namespaceSelector: + matchExpressions: + - key: ns + operator: In + values: + - x + - "y" + podSelector: + matchExpressions: + - key: pod + operator: In + values: + - b + - c + ports: + - port: 80 + protocol: TCP + podSelector: + matchLabels: + pod: a + policyTypes: + - Ingress + - Egress diff --git a/npm/pkg/dataplane/testdata/npmcachev1.json b/npm/pkg/dataplane/testdata/npmcachev1.json new file mode 100644 index 0000000000..1f522a4ac4 --- /dev/null +++ b/npm/pkg/dataplane/testdata/npmcachev1.json @@ -0,0 +1,517 @@ +{ + "ListMap": { + "azure-npm-1029213848": "component:tunnel", + "azure-npm-1160989932": "pod", + "azure-npm-1181855383": "version", + "azure-npm-1213884878": "namedport:serve-81-tcp", + "azure-npm-1593667296": "component", + "azure-npm-1698367069": "k8s-app", + "azure-npm-1883894896": "ns-kube-node-lease", + "azure-npm-2064349730": "ns-kube-system", + "azure-npm-2075916349": "namedport:serve-80-udp", + "azure-npm-2186870374": "ns-kube-public", + "azure-npm-2725615454": "kube-system", + "azure-npm-2837910840": "ns-y", + "azure-npm-2854688459": "ns-x", + "azure-npm-2888243697": "ns-z", + "azure-npm-2965211778": "namedport:metrics", + "azure-npm-2978150538": "kubernetes.io/cluster-service", + "azure-npm-3036558620": "pod-template-hash:774f99dbf4", + "azure-npm-3263818547": "k8s-app:coredns-autoscaler", + "azure-npm-33169192": "kubernetes.io/cluster-service:true", + "azure-npm-3350107434": "k8s-app:kube-dns", + "azure-npm-3495159168": "pod:b", + "azure-npm-3511936787": "pod:c", + "azure-npm-3545492025": "pod:a", + "azure-npm-3673508966": "k8s-app:metrics-server", + "azure-npm-3692662174": "namedport:serve-81-udp", + "azure-npm-3723426212": "pod-template-hash", + "azure-npm-4033764768": "base-in-ns-y-0in", + "azure-npm-4284971813": "namedport:serve-80-tcp", + "azure-npm-471829337": "version:v20", + "azure-npm-527074092": "app", + "azure-npm-71974944": "namedport:dns", + "azure-npm-784554818": "ns-default", + "azure-npm-814692531": "pod-template-hash:69c47794", + "azure-npm-819955173": "base-in-ns-y-0out", + "azure-npm-826519261": "app:konnectivity-agent", + "azure-npm-86579108": "pod-template-hash:7bf5785fc7", + "azure-npm-917915898": "namedport:dns-tcp", + "azure-npm-979220620": "pod-template-hash:5f85dc856b" + }, + "NsMap": { + "ns-default": { + "Name": "ns-default", + "LabelsMap": { + "kubernetes.io/metadata.name": "default" + } + }, + "ns-kube-node-lease": { + "Name": "ns-kube-node-lease", + "LabelsMap": { + "kubernetes.io/metadata.name": "kube-node-lease" + } + }, + "ns-kube-public": { + "Name": "ns-kube-public", + "LabelsMap": { + "kubernetes.io/metadata.name": "kube-public" + } + }, + "ns-kube-system": { + "Name": "ns-kube-system", + "LabelsMap": { + "addonmanager.kubernetes.io/mode": "Reconcile", + "control-plane": "true", + "kubernetes.io/cluster-service": "true", + "kubernetes.io/metadata.name": "kube-system" + } + }, + "ns-x": { + "Name": "ns-x", + "LabelsMap": { + "kubernetes.io/metadata.name": "x", + "ns": "x" + } + }, + "ns-y": { + "Name": "ns-y", + "LabelsMap": { + "kubernetes.io/metadata.name": "y", + "ns": "y" + } + }, + "ns-z": { + "Name": "ns-z", + "LabelsMap": { + "kubernetes.io/metadata.name": "z", + "ns": "z" + } + } + }, + "PodMap": { + "kube-system/coredns-69c47794-9vtmc": { + "Name": "coredns-69c47794-9vtmc", + "Namespace": "kube-system", + "PodIP": "10.224.0.41", + "Labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "pod-template-hash": "69c47794", + "version": "v20" + }, + "ContainerPorts": [ + { + "name": "dns", + "containerPort": 53, + "protocol": "UDP" + }, + { + "name": "dns-tcp", + "containerPort": 53, + "protocol": "TCP" + }, + { + "name": "metrics", + "containerPort": 9153, + "protocol": "TCP" + } + ], + "Phase": "Running" + }, + "kube-system/coredns-69c47794-c2987": { + "Name": "coredns-69c47794-c2987", + "Namespace": "kube-system", + "PodIP": "10.224.0.15", + "Labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "pod-template-hash": "69c47794", + "version": "v20" + }, + "ContainerPorts": [ + { + "name": "dns", + "containerPort": 53, + "protocol": "UDP" + }, + { + "name": "dns-tcp", + "containerPort": 53, + "protocol": "TCP" + }, + { + "name": "metrics", + "containerPort": 9153, + "protocol": "TCP" + } + ], + "Phase": "Running" + }, + "kube-system/coredns-autoscaler-5f85dc856b-m7rsz": { + "Name": "coredns-autoscaler-5f85dc856b-m7rsz", + "Namespace": "kube-system", + "PodIP": "10.224.0.88", + "Labels": { + "k8s-app": "coredns-autoscaler", + "pod-template-hash": "5f85dc856b" + }, + "ContainerPorts": [], + "Phase": "Running" + }, + "kube-system/konnectivity-agent-7bf5785fc7-dljxt": { + "Name": "konnectivity-agent-7bf5785fc7-dljxt", + "Namespace": "kube-system", + "PodIP": "10.224.0.39", + "Labels": { + "app": "konnectivity-agent", + "component": "tunnel", + "pod-template-hash": "7bf5785fc7" + }, + "ContainerPorts": [], + "Phase": "Running" + }, + "kube-system/konnectivity-agent-7bf5785fc7-khlww": { + "Name": "konnectivity-agent-7bf5785fc7-khlww", + "Namespace": "kube-system", + "PodIP": "10.224.0.92", + "Labels": { + "app": "konnectivity-agent", + "component": "tunnel", + "pod-template-hash": "7bf5785fc7" + }, + "ContainerPorts": [], + "Phase": "Running" + }, + "kube-system/metrics-server-774f99dbf4-ggb8w": { + "Name": "metrics-server-774f99dbf4-ggb8w", + "Namespace": "kube-system", + "PodIP": "10.224.0.80", + "Labels": { + "k8s-app": "metrics-server", + "pod-template-hash": "774f99dbf4" + }, + "ContainerPorts": [], + "Phase": "Running" + }, + "x/a": { + "Name": "a", + "Namespace": "x", + "PodIP": "10.224.0.87", + "Labels": { + "pod": "a" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "x/b": { + "Name": "b", + "Namespace": "x", + "PodIP": "10.224.0.20", + "Labels": { + "pod": "b" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "x/c": { + "Name": "c", + "Namespace": "x", + "PodIP": "10.224.0.40", + "Labels": { + "pod": "c" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "y/a": { + "Name": "a", + "Namespace": "y", + "PodIP": "10.224.0.70", + "Labels": { + "pod": "a" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "y/b": { + "Name": "b", + "Namespace": "y", + "PodIP": "10.224.0.17", + "Labels": { + "pod": "b" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "y/c": { + "Name": "c", + "Namespace": "y", + "PodIP": "10.224.0.43", + "Labels": { + "pod": "c" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "z/a": { + "Name": "a", + "Namespace": "z", + "PodIP": "10.224.0.68", + "Labels": { + "pod": "a" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "z/b": { + "Name": "b", + "Namespace": "z", + "PodIP": "10.224.0.13", + "Labels": { + "pod": "b" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "z/c": { + "Name": "c", + "Namespace": "z", + "PodIP": "10.224.0.42", + "Labels": { + "pod": "c" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + } + }, + "SetMap": { + "azure-npm-1029213848": "component:tunnel", + "azure-npm-1160989932": "pod", + "azure-npm-1181855383": "version", + "azure-npm-1213884878": "namedport:serve-81-tcp", + "azure-npm-1593667296": "component", + "azure-npm-1698367069": "k8s-app", + "azure-npm-1883894896": "ns-kube-node-lease", + "azure-npm-2064349730": "ns-kube-system", + "azure-npm-2075916349": "namedport:serve-80-udp", + "azure-npm-2186870374": "ns-kube-public", + "azure-npm-2725615454": "kube-system", + "azure-npm-2837910840": "ns-y", + "azure-npm-2854688459": "ns-x", + "azure-npm-2888243697": "ns-z", + "azure-npm-2965211778": "namedport:metrics", + "azure-npm-2978150538": "kubernetes.io/cluster-service", + "azure-npm-3036558620": "pod-template-hash:774f99dbf4", + "azure-npm-3263818547": "k8s-app:coredns-autoscaler", + "azure-npm-33169192": "kubernetes.io/cluster-service:true", + "azure-npm-3350107434": "k8s-app:kube-dns", + "azure-npm-3495159168": "pod:b", + "azure-npm-3511936787": "pod:c", + "azure-npm-3545492025": "pod:a", + "azure-npm-3673508966": "k8s-app:metrics-server", + "azure-npm-3692662174": "namedport:serve-81-udp", + "azure-npm-3723426212": "pod-template-hash", + "azure-npm-4033764768": "base-in-ns-y-0in", + "azure-npm-4284971813": "namedport:serve-80-tcp", + "azure-npm-471829337": "version:v20", + "azure-npm-527074092": "app", + "azure-npm-71974944": "namedport:dns", + "azure-npm-784554818": "ns-default", + "azure-npm-814692531": "pod-template-hash:69c47794", + "azure-npm-819955173": "base-in-ns-y-0out", + "azure-npm-826519261": "app:konnectivity-agent", + "azure-npm-86579108": "pod-template-hash:7bf5785fc7", + "azure-npm-917915898": "namedport:dns-tcp", + "azure-npm-979220620": "pod-template-hash:5f85dc856b" + } +} diff --git a/npm/pkg/dataplane/testdata/npmcachev2.json b/npm/pkg/dataplane/testdata/npmcachev2.json new file mode 100644 index 0000000000..f65b803020 --- /dev/null +++ b/npm/pkg/dataplane/testdata/npmcachev2.json @@ -0,0 +1,496 @@ +{ + "NodeName": "aks-nodepool1-74283791-vmss000001", + "NsMap": { + "default": { + "Name": "default", + "LabelsMap": { + "kubernetes.io/metadata.name": "default" + } + }, + "kube-node-lease": { + "Name": "kube-node-lease", + "LabelsMap": { + "kubernetes.io/metadata.name": "kube-node-lease" + } + }, + "kube-public": { + "Name": "kube-public", + "LabelsMap": { + "kubernetes.io/metadata.name": "kube-public" + } + }, + "kube-system": { + "Name": "kube-system", + "LabelsMap": { + "addonmanager.kubernetes.io/mode": "Reconcile", + "control-plane": "true", + "kubernetes.io/cluster-service": "true", + "kubernetes.io/metadata.name": "kube-system" + } + }, + "x": { + "Name": "x", + "LabelsMap": { + "kubernetes.io/metadata.name": "x", + "ns": "x" + } + }, + "y": { + "Name": "y", + "LabelsMap": { + "kubernetes.io/metadata.name": "y", + "ns": "y" + } + }, + "z": { + "Name": "z", + "LabelsMap": { + "kubernetes.io/metadata.name": "z", + "ns": "z" + } + } + }, + "PodMap": { + "kube-system/coredns-69c47794-9vtmc": { + "Name": "coredns-69c47794-9vtmc", + "Namespace": "kube-system", + "PodIP": "10.224.0.41", + "Labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "pod-template-hash": "69c47794", + "version": "v20" + }, + "ContainerPorts": [ + { + "name": "dns", + "containerPort": 53, + "protocol": "UDP" + }, + { + "name": "dns-tcp", + "containerPort": 53, + "protocol": "TCP" + }, + { + "name": "metrics", + "containerPort": 9153, + "protocol": "TCP" + } + ], + "Phase": "Running" + }, + "kube-system/coredns-69c47794-c2987": { + "Name": "coredns-69c47794-c2987", + "Namespace": "kube-system", + "PodIP": "10.224.0.15", + "Labels": { + "k8s-app": "kube-dns", + "kubernetes.io/cluster-service": "true", + "pod-template-hash": "69c47794", + "version": "v20" + }, + "ContainerPorts": [ + { + "name": "dns", + "containerPort": 53, + "protocol": "UDP" + }, + { + "name": "dns-tcp", + "containerPort": 53, + "protocol": "TCP" + }, + { + "name": "metrics", + "containerPort": 9153, + "protocol": "TCP" + } + ], + "Phase": "Running" + }, + "kube-system/coredns-autoscaler-5f85dc856b-m7rsz": { + "Name": "coredns-autoscaler-5f85dc856b-m7rsz", + "Namespace": "kube-system", + "PodIP": "10.224.0.88", + "Labels": { + "k8s-app": "coredns-autoscaler", + "pod-template-hash": "5f85dc856b" + }, + "ContainerPorts": [], + "Phase": "Running" + }, + "kube-system/konnectivity-agent-7bf5785fc7-dljxt": { + "Name": "konnectivity-agent-7bf5785fc7-dljxt", + "Namespace": "kube-system", + "PodIP": "10.224.0.39", + "Labels": { + "app": "konnectivity-agent", + "component": "tunnel", + "pod-template-hash": "7bf5785fc7" + }, + "ContainerPorts": [], + "Phase": "Running" + }, + "kube-system/konnectivity-agent-7bf5785fc7-khlww": { + "Name": "konnectivity-agent-7bf5785fc7-khlww", + "Namespace": "kube-system", + "PodIP": "10.224.0.92", + "Labels": { + "app": "konnectivity-agent", + "component": "tunnel", + "pod-template-hash": "7bf5785fc7" + }, + "ContainerPorts": [], + "Phase": "Running" + }, + "kube-system/metrics-server-774f99dbf4-ggb8w": { + "Name": "metrics-server-774f99dbf4-ggb8w", + "Namespace": "kube-system", + "PodIP": "10.224.0.80", + "Labels": { + "k8s-app": "metrics-server", + "pod-template-hash": "774f99dbf4" + }, + "ContainerPorts": [], + "Phase": "Running" + }, + "x/a": { + "Name": "a", + "Namespace": "x", + "PodIP": "10.224.0.87", + "Labels": { + "pod": "a" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "x/b": { + "Name": "b", + "Namespace": "x", + "PodIP": "10.224.0.20", + "Labels": { + "pod": "b" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "x/c": { + "Name": "c", + "Namespace": "x", + "PodIP": "10.224.0.40", + "Labels": { + "pod": "c" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "y/a": { + "Name": "a", + "Namespace": "y", + "PodIP": "10.224.0.70", + "Labels": { + "pod": "a" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "y/b": { + "Name": "b", + "Namespace": "y", + "PodIP": "10.224.0.17", + "Labels": { + "pod": "b" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "y/c": { + "Name": "c", + "Namespace": "y", + "PodIP": "10.224.0.43", + "Labels": { + "pod": "c" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "z/a": { + "Name": "a", + "Namespace": "z", + "PodIP": "10.224.0.68", + "Labels": { + "pod": "a" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "z/b": { + "Name": "b", + "Namespace": "z", + "PodIP": "10.224.0.13", + "Labels": { + "pod": "b" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + }, + "z/c": { + "Name": "c", + "Namespace": "z", + "PodIP": "10.224.0.42", + "Labels": { + "pod": "c" + }, + "ContainerPorts": [ + { + "name": "serve-80-tcp", + "containerPort": 80, + "protocol": "TCP" + }, + { + "name": "serve-80-udp", + "containerPort": 80, + "protocol": "UDP" + }, + { + "name": "serve-81-tcp", + "containerPort": 81, + "protocol": "TCP" + }, + { + "name": "serve-81-udp", + "containerPort": 81, + "protocol": "UDP" + } + ], + "Phase": "Running" + } + }, + "SetMap": { + "azure-npm-107991907": "podlabel-k8s-app:coredns-autoscaler", + "azure-npm-1213884878": "namedport:serve-81-tcp", + "azure-npm-1228747572": "podlabel-pod-template-hash:7bf5785fc7", + "azure-npm-1343132199": "podlabel-version", + "azure-npm-1385180724": "podlabel-pod-template-hash", + "azure-npm-1529935048": "podlabel-component:tunnel", + "azure-npm-1639206293": "nslabel-all-namespaces", + "azure-npm-1802501696": "nslabel-kubernetes.io/cluster-service", + "azure-npm-1883894896": "ns-kube-node-lease", + "azure-npm-1889013859": "podlabel-pod-template-hash:69c47794", + "azure-npm-1923986458": "podlabel-kubernetes.io/cluster-service", + "azure-npm-1977654781": "nslabel-kubernetes.io/metadata.name:kube-system", + "azure-npm-2064349730": "ns-kube-system", + "azure-npm-2075916349": "namedport:serve-80-udp", + "azure-npm-2095721080": "nslabel-ns:z", + "azure-npm-2129276318": "nslabel-ns:x", + "azure-npm-2146053937": "nslabel-ns:y", + "azure-npm-2186870374": "ns-kube-public", + "azure-npm-2261148835": "nslabel-control-plane", + "azure-npm-2293485820": "podlabel-pod", + "azure-npm-2540899149": "podlabel-k8s-app", + "azure-npm-2547206700": "podlabel-pod-template-hash:774f99dbf4", + "azure-npm-2647803239": "nslabel-control-plane:true", + "azure-npm-2682470511": "nestedlabel-pod:a:b", + "azure-npm-2714724634": "podlabel-k8s-app:kube-dns", + "azure-npm-2764516068": "nslabel-addonmanager.kubernetes.io/mode", + "azure-npm-2837910840": "ns-y", + "azure-npm-2854688459": "ns-x", + "azure-npm-2888243697": "ns-z", + "azure-npm-2937511974": "nslabel-kubernetes.io/metadata.name", + "azure-npm-2965211778": "namedport:metrics", + "azure-npm-2977804918": "podlabel-k8s-app:metrics-server", + "azure-npm-3059925008": "podlabel-component", + "azure-npm-3548820133": "nslabel-kubernetes.io/metadata.name:kube-public", + "azure-npm-3692662174": "namedport:serve-81-udp", + "azure-npm-3717707100": "podlabel-app", + "azure-npm-3731293995": "nslabel-kubernetes.io/metadata.name:kube-node-lease", + "azure-npm-3872074864": "podlabel-pod:b", + "azure-npm-3888852483": "podlabel-pod:c", + "azure-npm-3922407721": "podlabel-pod:a", + "azure-npm-397063964": "podlabel-pod-template-hash:5f85dc856b", + "azure-npm-3993150488": "podlabel-kubernetes.io/cluster-service:true", + "azure-npm-3999554562": "nslabel-kubernetes.io/cluster-service:true", + "azure-npm-406267145": "nslabel-kubernetes.io/metadata.name:default", + "azure-npm-4150775300": "nslabel-kubernetes.io/metadata.name:x", + "azure-npm-4167552919": "nslabel-kubernetes.io/metadata.name:y", + "azure-npm-4184330538": "nslabel-kubernetes.io/metadata.name:z", + "azure-npm-4272224941": "podlabel-app:konnectivity-agent", + "azure-npm-4284971813": "namedport:serve-80-tcp", + "azure-npm-483924252": "nslabel-ns", + "azure-npm-55798953": "nestedlabel-pod:b:c", + "azure-npm-708060905": "podlabel-version:v20", + "azure-npm-71974944": "namedport:dns", + "azure-npm-784554818": "ns-default", + "azure-npm-917915898": "namedport:dns-tcp", + "azure-npm-984584486": "nslabel-addonmanager.kubernetes.io/mode:Reconcile" + } +} diff --git a/npm/pkg/dataplane/types.go b/npm/pkg/dataplane/types.go index 3385487958..cab116ced1 100644 --- a/npm/pkg/dataplane/types.go +++ b/npm/pkg/dataplane/types.go @@ -9,7 +9,7 @@ import ( type GenericDataplane interface { BootupDataplane() error RunPeriodicTasks() - GetAllIPSets() []string + GetAllIPSets() map[string]string GetIPSet(setName string) *ipsets.IPSet CreateIPSets(setMetadatas []*ipsets.IPSetMetadata) DeleteIPSet(setMetadata *ipsets.IPSetMetadata, deleteOption util.DeleteOption) diff --git a/npm/util/ioutil/restore.go b/npm/util/ioutil/restore.go index 898d15f6b6..4bfaebebe3 100644 --- a/npm/util/ioutil/restore.go +++ b/npm/util/ioutil/restore.go @@ -141,7 +141,7 @@ func (creator *FileCreator) RunCommandWithFile(cmd string, args ...string) error if wasFileAltered { sameNew = "updated" } - msg := fmt.Sprintf("on try number %d, failed to run command [%s]. Rerunning with %s file. Had error [%s].\nUsed file:\n%s", creator.tryCount, commandString, sameNew, err.Error(), fileString) + msg := fmt.Sprintf("on try number %d, failed to run command [%s]. Rerunning with %s file. Had error [%s].Used file:%s", creator.tryCount, commandString, sameNew, err.Error(), fileString) klog.Error(msg) metrics.SendErrorLogAndMetric(util.UtilID, "error: %s", msg)