From 4e429597dee64e9dc65cc1cdfb5a4b590d9b92fc Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Thu, 12 Mar 2020 10:07:57 -0700 Subject: [PATCH 1/6] About to take debugging comments out --- Makefile | 25 +++- cni/azure-linux.conflist | 3 + cni/netconfig.go | 5 + cni/network/network.go | 4 + cni/network/network_linux.go | 43 +++++++ cni/network/network_windows.go | 3 + cnms/Dockerfile | 6 + cnms/cnmspackage/api.go | 6 + cnms/cnmspackage/monitor2rules_linux.go | 152 +++++++++++++++++++++++ cnms/service/networkmonitor.go | 153 ++++++++++++++++++++++++ cnms/service/networkmonitor_test.go | 113 +++++++++++++++++ common/config.go | 5 + ebtables/ebtables.go | 147 +++++++++++------------ move-to-cluster.sh | 4 + network/manager.go | 10 ++ network/monitor_linux.go | 63 ++++++++++ network/monitor_windows.go | 3 + networkmonitor/Dockerfile | 6 + networkmonitor/main.go | 153 ++++++++++++++++++++++++ 19 files changed, 828 insertions(+), 76 deletions(-) create mode 100644 cnms/Dockerfile create mode 100644 cnms/cnmspackage/api.go create mode 100644 cnms/cnmspackage/monitor2rules_linux.go create mode 100644 cnms/service/networkmonitor.go create mode 100644 cnms/service/networkmonitor_test.go create mode 100755 move-to-cluster.sh create mode 100644 network/monitor_linux.go create mode 100644 network/monitor_windows.go create mode 100644 networkmonitor/Dockerfile create mode 100644 networkmonitor/main.go diff --git a/Makefile b/Makefile index 183977dcdc..3b87f6d678 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,12 @@ CNSFILES = \ $(COREFILES) \ $(CNMFILES) +CNMSFILES = \ + $(wildcard cnms/*.go) \ + $(wildcard cnms/service/*.go) \ + $(wildcard cnms/cnmspackage/*.go) \ + $(COREFILES) + NPMFILES = \ $(wildcard npm/*.go) \ $(wildcard npm/ipsm/*.go) \ @@ -71,6 +77,7 @@ CNI_IPAM_DIR = cni/ipam/plugin CNI_TELEMETRY_DIR = cni/telemetry/service TELEMETRY_CONF_DIR = telemetry CNS_DIR = cns/service +CNMS_DIR = cnms/service NPM_DIR = npm/plugin OUTPUT_DIR = output BUILD_DIR = $(OUTPUT_DIR)/$(GOOS)_$(GOARCH) @@ -78,6 +85,7 @@ CNM_BUILD_DIR = $(BUILD_DIR)/cnm CNI_BUILD_DIR = $(BUILD_DIR)/cni CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy CNS_BUILD_DIR = $(BUILD_DIR)/cns +CNMS_BUILD_DIR = $(BUILD_DIR)/cnms NPM_BUILD_DIR = $(BUILD_DIR)/npm NPM_TELEMETRY_DIR = $(NPM_BUILD_DIR)/telemetry CNI_AI_ID = 5515a1eb-b2bc-406a-98eb-ba462e6f0411 @@ -107,6 +115,7 @@ CNM_ARCHIVE_NAME = azure-vnet-cnm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNI_ARCHIVE_NAME = azure-vnet-cni-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNI_MULTITENANCY_ARCHIVE_NAME = azure-vnet-cni-multitenancy-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNS_ARCHIVE_NAME = azure-cns-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) +CNMS_ARCHIVE_NAME = azure-cnms-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) NPM_ARCHIVE_NAME = azure-npm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) NPM_IMAGE_ARCHIVE_NAME = azure-npm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) TELEMETRY_IMAGE_ARCHIVE_NAME = azure-vnet-telemetry-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) @@ -140,11 +149,12 @@ azure-vnet-telemetry: $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) # Azure-NPM only supports Linux for now. ifeq ($(GOOS),linux) +azure-cnms: $(CNMS_BUILD_DIR)/azure-cnms$(EXE_EXT) cnms-archive azure-npm: $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT) npm-archive endif ifeq ($(GOOS),linux) -all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-npm +all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-cnms azure-npm else all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns endif @@ -181,6 +191,10 @@ $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT): $(CNIFILES) $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT): $(CNSFILES) go build -v -o $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -X $(cnsaipath)=$(CNS_AI_ID) -s -w" $(CNS_DIR)/*.go +# Build the Azure CNMS Service. +$(CNMS_BUILD_DIR)/azure-cnms$(EXE_EXT): $(CNMSFILES) + go build -v -o $(CNMS_BUILD_DIR)/azure-cnms$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNMS_DIR)/*.go + # Build the Azure NPM plugin. $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT): $(NPMFILES) go build -v -o $(NPM_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNI_TELEMETRY_DIR)/*.go @@ -325,6 +339,15 @@ cns-archive: cd $(CNS_BUILD_DIR) && $(ARCHIVE_CMD) $(CNS_ARCHIVE_NAME) azure-cns$(EXE_EXT) cns_config.json chown $(BUILD_USER):$(BUILD_USER) $(CNS_BUILD_DIR)/$(CNS_ARCHIVE_NAME) +# Create a CNMS archive for the target platform. Only Linux is supported for now. +.PHONY: cnms-archive +cnms-archive: +ifeq ($(GOOS),linux) + chmod 0755 $(CNMS_BUILD_DIR)/azure-cnms$(EXE_EXT) + cd $(CNMS_BUILD_DIR) && $(ARCHIVE_CMD) $(CNMS_ARCHIVE_NAME) azure-cnms$(EXE_EXT) + chown $(BUILD_USER):$(BUILD_USER) $(CNMS_BUILD_DIR)/$(CNMS_ARCHIVE_NAME) +endif + # Create a NPM archive for the target platform. Only Linux is supported for now. .PHONY: npm-archive npm-archive: diff --git a/cni/azure-linux.conflist b/cni/azure-linux.conflist index fad44fb448..4793e49d1c 100644 --- a/cni/azure-linux.conflist +++ b/cni/azure-linux.conflist @@ -9,6 +9,9 @@ "ipsToRouteViaHost":["169.254.20.10"], "ipam":{ "type":"azure-vnet-ipam" + }, + "networkMonitor":{ + "name":"azure-cnms" } }, { diff --git a/cni/netconfig.go b/cni/netconfig.go index 54a1c13995..5d3142b355 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -67,6 +67,11 @@ type NetworkConfig struct { Address string `json:"ipAddress,omitempty"` QueryInterval string `json:"queryInterval,omitempty"` } + NetworkMonitor struct { + Name string `json:"name"` + Interval int `json:"interval,omitempty"` + Disable bool `json:"disable,omitempty"` + } DNS cniTypes.DNS `json:"dns"` RuntimeConfig RuntimeConfig `json:"runtimeConfig"` AdditionalArgs []KVPair diff --git a/cni/network/network.go b/cni/network/network.go index 35a7f0881c..5400cb3282 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -263,6 +263,10 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Read network configuration %+v.", nwCfg) + if err := startMonitorIfNotRunning(nwCfg); err != nil { + log.Printf("Starting network monitor failed with %v", err) + } + // Temporary if block to determing whether we disable SNAT on host (for multi-tenant scenario only) if nwCfg.MultiTenancy { if enableSnatForDns, nwCfg.EnableSnatOnHost, err = determineSnat(); err != nil { diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 263a30ae31..a375568457 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -1,7 +1,9 @@ package network import ( + "fmt" "net" + "os/exec" "strconv" "github.com/Azure/azure-container-networking/cni" @@ -9,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/policy" + "github.com/Azure/azure-container-networking/platform" cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypes "github.com/containernetworking/cni/pkg/types" @@ -18,6 +21,7 @@ import ( const ( snatInterface = "eth1" infraInterface = "eth2" + binPath = "/opt/cni/bin" ) const ( @@ -85,6 +89,45 @@ func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { } } +func startMonitorIfNotRunning(nwCfg *cni.NetworkConfig) error { + log.Printf("startMonitorIfNotRunning function called") + + if nwCfg != nil { + + netmon := nwCfg.NetworkMonitor + if netmon.Name == "" { + return fmt.Errorf("NetworkMonitor is not set in conflist") + } + + if netmon.Disable == true { + log.Printf("Network Monitor is not enabled") + return nil + } + + cmd := fmt.Sprintf("pgrep -x %v", netmon.Name) + _, err := platform.ExecuteCommand(cmd) + if err == nil { + log.Printf("Azure Network monitor is already running") + } + + cmd = fmt.Sprintf(binPath+"/%v", netmon.Name) + + if netmon.Interval != 0 { + cmd = fmt.Sprintf("%v -it %v", cmd, netmon.Interval) + } + + startCmd := exec.Command("sh", "-c", cmd) + if err := startCmd.Start(); err != nil { + log.Printf("startcmd failed with error %v", err) + return err + } + + log.Printf("Azure Network Monitor started") + } + + return nil +} + func setupInfraVnetRoutingForMultitenancy( nwCfg *cni.NetworkConfig, azIpamResult *cniTypesCurr.Result, diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 615c871496..481437b0e6 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -118,6 +118,9 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { } +func startMonitorIfNotRunning(nwCfg *cni.NetworkConfig) error { +} + func updateSubnetPrefix(cnsNwConfig *cns.GetNetworkContainerResponse, subnetPrefix *net.IPNet) error { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { ipconfig := cnsNwConfig.IPConfiguration diff --git a/cnms/Dockerfile b/cnms/Dockerfile new file mode 100644 index 0000000000..21a2c1adab --- /dev/null +++ b/cnms/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:latest +RUN apt -y update +RUN apt install -y ebtables +RUN apt install -y net-tools +COPY networkmonitor /usr/bin/networkmonitor +CMD ["/usr/bin/networkmonitor"] \ No newline at end of file diff --git a/cnms/cnmspackage/api.go b/cnms/cnmspackage/api.go new file mode 100644 index 0000000000..01979756e4 --- /dev/null +++ b/cnms/cnmspackage/api.go @@ -0,0 +1,6 @@ +package cnms + +type NetworkMonitor struct { + AddRulesToBeValidated map[string]int + DeleteRulesToBeValidated map[string]int +} diff --git a/cnms/cnmspackage/monitor2rules_linux.go b/cnms/cnmspackage/monitor2rules_linux.go new file mode 100644 index 0000000000..1903b5c144 --- /dev/null +++ b/cnms/cnmspackage/monitor2rules_linux.go @@ -0,0 +1,152 @@ +package cnms + +import ( + "github.com/Azure/azure-container-networking/ebtables" + "github.com/Azure/azure-container-networking/log" +) + +func (networkMonitor *NetworkMonitor) deleteRulesNotExistInMap(chainRules map[string]string, stateRules map[string]string) { + log.Printf("deleteRulesNotExistInMap function called\n") + table := "nat" + action := ebtables.Delete + //Here I think + for rule, chain := range chainRules { + if _, ok := stateRules[rule]; !ok { + if itr, ok := networkMonitor.DeleteRulesToBeValidated[rule]; ok && itr > 0 { + log.Printf("[monitor] Deleting Ebtable rule as it didn't exist in state for %d iterations chain %v rule %v", itr, chain, rule) + if err := ebtables.SetEbRule(table, action, chain, rule); err != nil { + log.Printf("[monitor] Error while deleting ebtable rule %v", err) + } + + delete(networkMonitor.DeleteRulesToBeValidated, rule) + } else { + log.Printf("[DELETE] Found unmatched rule chain %v rule %v itr %d. Giving one more iteration", chain, rule, itr) + networkMonitor.DeleteRulesToBeValidated[rule] = itr + 1 + } + } + } +} + +func deleteRulesExistInMap(originalChainRules map[string]string, stateRules map[string]string) { + log.Printf("deleteRulesExistinMap funciton called\n") + table := "nat" + action := ebtables.Delete + + for rule, chain := range originalChainRules { + if _, ok := stateRules[rule]; ok { + log.Printf("[monitor] Deleting Ebtable rule which existed in map %v", rule) + if err := ebtables.SetEbRule(table, action, chain, rule); err != nil { + log.Printf("[monitor] Error while deleting ebtable rule %v", err) + } + } + } +} + +func (networkMonitor *NetworkMonitor) addRulesNotExistInMap( + stateRules map[string]string, + chainRules map[string]string) { + log.Printf("addRulesNotExistInMap funciton called \n") + + table := "nat" + action := ebtables.Append + + for rule, chain := range stateRules { + if _, ok := chainRules[rule]; !ok { + if itr, ok := networkMonitor.AddRulesToBeValidated[rule]; ok && itr > 0 { + log.Printf("[monitor] Adding Ebtable rule as it didn't exist in state for %d iterations chain %v rule %v", itr, chain, rule) + if err := ebtables.SetEbRule(table, action, chain, rule); err != nil { + log.Printf("[monitor] Error while adding ebtable rule %v", err) + } + + delete(networkMonitor.AddRulesToBeValidated, rule) + } else { + log.Printf("[ADD] Found unmatched rule chain %v rule %v itr %d. Giving one more iteration", chain, rule, itr) + networkMonitor.AddRulesToBeValidated[rule] = itr + 1 + } + } + } +} + +func (networkMonitor *NetworkMonitor) CreateRequiredL2Rules( + currentEbtableRulesMap map[string]string, + currentStateRulesMap map[string]string) error { + log.Printf("CreateRequiredL2Rules function called\n") + + for rule := range networkMonitor.AddRulesToBeValidated { + log.Printf("Rule in AddRulesToBeValidated %v", rule) + if _, ok := currentStateRulesMap[rule]; !ok { + log.Printf("Deleting Rule") + delete(networkMonitor.AddRulesToBeValidated, rule) + } + } + + networkMonitor.addRulesNotExistInMap(currentStateRulesMap, currentEbtableRulesMap) + + return nil +} + +func (networkMonitor *NetworkMonitor) RemoveInvalidL2Rules( + currentEbtableRulesMap map[string]string, + currentStateRulesMap map[string]string) error { + log.Printf("RemoveInvalidL2Rules function called\n") + + for rule := range networkMonitor.DeleteRulesToBeValidated { + log.Printf("Checking DeleteRulesToBeValidated rule: %v", rule) + if _, ok := currentEbtableRulesMap[rule]; !ok { + log.Printf("DeleteRulesToBeValidated deleting rule: %v", rule) + delete(networkMonitor.DeleteRulesToBeValidated, rule) + } + } + + // originalChainRules := make(map[string]string) + + // if err := generateL2RulesMap(originalChainRules, ebtables.PreRouting); err != nil { + // return err + // } + + // if err := generateL2RulesMap(originalChainRules, ebtables.PostRouting); err != nil { + // return err + // } + + // deleteRulesExistInMap(originalChainRules, currentEbtableRulesMap) + + networkMonitor.deleteRulesNotExistInMap(currentEbtableRulesMap, currentStateRulesMap) + + return nil +} + +func generateL2RulesMap(currentEbtableRulesMap map[string]string, chainName string) error { + log.Printf("generateL2RulesMap funciton called\n") + table := "nat" + rules, err := ebtables.GetEbtableRules(table, chainName) + + if err != nil { + log.Printf("[monitor] Error while getting rules list from table %v chain %v. Error: %v", + table, chainName, err) + return err + } + + log.Printf("[monitor] Rules count : %v", len(rules)) + + for _, rule := range rules { + log.Printf("[monitor] Adding rule %s mapped to chainName %s.", rule, chainName) + currentEbtableRulesMap[rule] = chainName + } + + return nil +} + +func GetEbTableRulesInMap() (map[string]string, error) { + log.Printf("GetEbtableRulesInMap funciton called \n") + currentEbtableRulesMap := make(map[string]string) + + if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PreRouting); err != nil { + return nil, err + } + + if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PostRouting); err != nil { + return nil, err + } + + return currentEbtableRulesMap, nil +} diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go new file mode 100644 index 0000000000..9ce72d1d5b --- /dev/null +++ b/cnms/service/networkmonitor.go @@ -0,0 +1,153 @@ +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package main + +import ( + "fmt" + "os" + "time" + + cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" + acn "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network" + "github.com/Azure/azure-container-networking/platform" + "github.com/Azure/azure-container-networking/store" +) + +const ( + // Service name. + name = "azure-cnimonitor" + pluginName = "azure-vnet" + DEFAULT_TIMEOUT_IN_SECS = "10" + DefaultMode = "AzureChain" +) + +// Version is populated by make during build. +var version string + +// Command line arguments for CNM plugin. +var args = acn.ArgumentList{ + { + Name: acn.OptLogLevel, + Shorthand: acn.OptLogLevelAlias, + Description: "Set the logging level", + Type: "int", + DefaultValue: acn.OptLogLevelInfo, + ValueMap: map[string]interface{}{ + acn.OptLogLevelInfo: log.LevelInfo, + acn.OptLogLevelDebug: log.LevelDebug, + }, + }, + { + Name: acn.OptLogTarget, + Shorthand: acn.OptLogTargetAlias, + Description: "Set the logging target", + Type: "int", + DefaultValue: acn.OptLogTargetFile, + ValueMap: map[string]interface{}{ + acn.OptLogTargetSyslog: log.TargetSyslog, + acn.OptLogTargetStderr: log.TargetStderr, + acn.OptLogTargetFile: log.TargetLogfile, + }, + }, + { + Name: acn.OptLogLocation, + Shorthand: acn.OptLogLocationAlias, + Description: "Set the directory location where logs will be saved", + Type: "string", + DefaultValue: "", + }, + { + Name: acn.OptIntervalTime, + Shorthand: acn.OptIntervalTimeAlias, + Description: "Periodic Interval Time", + Type: "int", + DefaultValue: DEFAULT_TIMEOUT_IN_SECS, + }, + { + Name: acn.OptVersion, + Shorthand: acn.OptVersionAlias, + Description: "Print version information", + Type: "bool", + DefaultValue: false, + }, +} + +// Prints description and version information. +func printVersion() { + fmt.Printf("Azure Container Network Monitoring Service\n") + fmt.Printf("Version %v\n", version) +} + +// Main is the entry point for CNMS. +func main() { + // Initialize and parse command line arguments. + acn.ParseArgs(&args, printVersion) + + logLevel := acn.GetArg(acn.OptLogLevel).(int) + logTarget := acn.GetArg(acn.OptLogTarget).(int) + logDirectory := acn.GetArg(acn.OptLogLocation).(string) + timeout := acn.GetArg(acn.OptIntervalTime).(int) + vers := acn.GetArg(acn.OptVersion).(bool) + + if vers { + printVersion() + os.Exit(0) + } + + // Initialize CNMS. + var config acn.PluginConfig + config.Version = version + + // Create a channel to receive unhandled errors from CNMS. + config.ErrChan = make(chan error, 1) + + var err error + // Create logging provider. + log.SetName(name) + log.SetLevel(logLevel) + if err := log.SetTargetLogDirectory(logTarget, logDirectory); err != nil { + fmt.Printf("[netmon] Failed to configure logging: %v\n", err) + return + } + + // Log platform information. + log.Printf("[netmon] Running on %v", platform.GetOSInfo()) + + netMonitor := &cnms.NetworkMonitor{ + AddRulesToBeValidated: make(map[string]int), + DeleteRulesToBeValidated: make(map[string]int), + } + + for true { + config.Store, err = store.NewJsonFileStore(platform.CNIRuntimePath + pluginName + ".json") + if err != nil { + fmt.Printf("[netmon] Failed to create store: %v\n", err) + return + } + + nm, err := network.NewNetworkManager() + if err != nil { + log.Printf("[netmon] Failed while creating network manager") + return + } + + if err := nm.Initialize(&config); err != nil { + log.Printf("[netmon] Failed while initializing network manager %+v", err) + } + + log.Printf("[netmon] network manager:%+v", nm) + + if err := nm.SetupNetworkUsingState(netMonitor); err != nil { + log.Printf("[netmon] Failed while calling SetupNetworkUsingState with error %v", err) + } + + log.Printf("[netmon] Going to sleep for %v seconds", timeout) + time.Sleep(time.Duration(timeout) * time.Second) + nm = nil + } + + log.Close() +} diff --git a/cnms/service/networkmonitor_test.go b/cnms/service/networkmonitor_test.go new file mode 100644 index 0000000000..760ccfa23c --- /dev/null +++ b/cnms/service/networkmonitor_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "os" + "testing" + + cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" + "github.com/Azure/azure-container-networking/ebtables" +) + +const ( + test1 = "test1.json" + test2 = "test2.json" +) + +var stateMapkey []string + +func TestMain(m *testing.M) { + stateMapkey = append(stateMapkey, "-p ARP -i eth0 --arp-op Reply -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT") + stateMapkey = append(stateMapkey, "-p ARP --arp-op Request --arp-ip-dst 10.240.0.6 -j arpreply --arpreply-mac cc:ad:1d:4e:e5:f1") + stateMapkey = append(stateMapkey, "-p IPv4 -i eth0 --ip-dst 10.240.0.6 -j dnat --to-dst cc:ad:1d:4e:e5:f1 --dnat-target ACCEPT") + exitCode := m.Run() + os.Exit(exitCode) +} + +func addStateRulesToMap() map[string]string { + rulesMap := make(map[string]string) + + for _, value := range stateMapkey { + rulesMap[value] = ebtables.PreRouting + } + + rulesMap["-s Unicast -o eth0 -j snat --to-src 00:0d:12:3a:5d:32 --snat-arp --snat-target ACCEPT"] = ebtables.PostRouting + + return rulesMap +} + +func TestAddMissingRule(t *testing.T) { + netMonitor := &cnms.NetworkMonitor{ + AddRulesToBeValidated: make(map[string]int), + DeleteRulesToBeValidated: make(map[string]int), + } + + currentStateRulesMap := addStateRulesToMap() + currentEbTableRulesMap := make(map[string]string) + testKey := "" + + for key, value := range currentStateRulesMap { + if value != ebtables.PostRouting { + currentEbTableRulesMap[key] = value + } else { + testKey = key + } + } + + netMonitor.CreateRequiredL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.AddRulesToBeValidated) != 1 { + t.Fatalf("Expected AddRulesToBeValidated length to be 1 but got %v", len(netMonitor.AddRulesToBeValidated)) + } + + netMonitor.RemoveInvalidL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.DeleteRulesToBeValidated) != 0 { + t.Fatalf("Expected DeleteRulesToBeValidated length to be 0 but got %v", len(netMonitor.DeleteRulesToBeValidated)) + } + + for key, value := range netMonitor.AddRulesToBeValidated { + if key != testKey { + t.Fatalf("Expected AzurePostRouting snat but got %v", value) + } + } + + netMonitor.CreateRequiredL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.AddRulesToBeValidated) != 0 { + t.Fatalf("Expected AddRulesToBeValidated length to be 0 but got %v", len(netMonitor.AddRulesToBeValidated)) + } +} + +func TestDeleteInvalidRule(t *testing.T) { + netMonitor := &cnms.NetworkMonitor{ + AddRulesToBeValidated: make(map[string]int), + DeleteRulesToBeValidated: make(map[string]int), + } + + currentStateRulesMap := addStateRulesToMap() + currentEbTableRulesMap := make(map[string]string) + + for key, value := range currentStateRulesMap { + currentEbTableRulesMap[key] = value + } + + delete(currentStateRulesMap, stateMapkey[0]) + + netMonitor.CreateRequiredL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.AddRulesToBeValidated) != 0 { + t.Fatalf("Expected AddRulesToBeValidated length to be 0 but got %v", len(netMonitor.AddRulesToBeValidated)) + } + + netMonitor.RemoveInvalidL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.DeleteRulesToBeValidated) != 1 { + t.Fatalf("Expected DeleteRulesToBeValidated length to be 1 but got %v", len(netMonitor.DeleteRulesToBeValidated)) + } + + for key, value := range netMonitor.AddRulesToBeValidated { + if key != stateMapkey[0] { + t.Fatalf("Expected %v but got %v", stateMapkey[0], value) + } + } + + netMonitor.RemoveInvalidL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.DeleteRulesToBeValidated) != 0 { + t.Fatalf("Expected DeleteRulesToBeValidated length to be 0 but got %v", len(netMonitor.DeleteRulesToBeValidated)) + } +} diff --git a/common/config.go b/common/config.go index a7f5d42330..71f0495926 100644 --- a/common/config.go +++ b/common/config.go @@ -53,6 +53,11 @@ const ( OptReportToHostInterval = "report-interval" OptReportToHostIntervalAlias = "hostinterval" + // Periodic Interval Time + OptIntervalTime = "interval" + OptIntervalTimeAlias = "it" + OptDefaultIntervalTime = "defaultinterval" + // Version. OptVersion = "version" OptVersionAlias = "v" diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go index ef516d42a7..d241d11719 100644 --- a/ebtables/ebtables.go +++ b/ebtables/ebtables.go @@ -5,12 +5,9 @@ package ebtables import ( "fmt" - "io/ioutil" "net" - "os/exec" "strings" - "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" ) @@ -18,48 +15,88 @@ const ( // Ebtables actions. Append = "-A" Delete = "-D" + // Ebtables + Nat = "nat" + Broute = "broute" + // Ebtable chains + PreRouting = "PREROUTING" + PostRouting = "POSTROUTING" + Brouting = "BROUTING" ) -// InstallEbtables installs the ebtables package. -func installEbtables() { - version, _ := ioutil.ReadFile("/proc/version") - os := strings.ToLower(string(version)) - - if strings.Contains(os, "ubuntu") { - executeShellCommand("apt-get install ebtables") - } else if strings.Contains(os, "redhat") { - executeShellCommand("yum install ebtables") - } else { - log.Printf("Unable to detect OS platform. Please make sure the ebtables package is installed.") - } -} - // SetSnatForInterface sets a MAC SNAT rule for an interface. func SetSnatForInterface(interfaceName string, macAddress net.HardwareAddr, action string) error { - command := fmt.Sprintf( - "ebtables -t nat %s POSTROUTING -s unicast -o %s -j snat --to-src %s --snat-arp --snat-target ACCEPT", - action, interfaceName, macAddress.String()) + table := Nat + chain := PostRouting + rule := fmt.Sprintf("-s unicast -o %s -j snat --to-src %s --snat-arp --snat-target ACCEPT", + interfaceName, macAddress.String()) - return executeShellCommand(command) + return runEbCmd(table, action, chain, rule) } // SetArpReply sets an ARP reply rule for the given target IP address and MAC address. func SetArpReply(ipAddress net.IP, macAddress net.HardwareAddr, action string) error { - command := fmt.Sprintf( - "ebtables -t nat %s PREROUTING -p ARP --arp-op Request --arp-ip-dst %s -j arpreply --arpreply-mac %s --arpreply-target DROP", - action, ipAddress, macAddress.String()) + table := Nat + chain := PreRouting + rule := fmt.Sprintf("-p ARP --arp-op Request --arp-ip-dst %s -j arpreply --arpreply-mac %s --arpreply-target DROP", + ipAddress, macAddress.String()) - return executeShellCommand(command) + return runEbCmd(table, action, chain, rule) } // SetBrouteAccept sets an EB rule. func SetBrouteAccept(ipAddress, action string) error { - command := fmt.Sprintf( - "ebtables -t broute %s BROUTING --ip-dst %s -p IPv4 -j redirect --redirect-target ACCEPT", - action, ipAddress) - _, err := platform.ExecuteCommand(command) + table := Broute + chain := Brouting + rule := fmt.Sprintf("--ip-dst %s -p IPv4 -j redirect --redirect-target ACCEPT", ipAddress) - return err + return runEbCmd(table, action, chain, rule) +} + +// SetDnatForArpReplies sets a MAC DNAT rule for ARP replies received on an interface. +func SetDnatForArpReplies(interfaceName string, action string) error { + table := Nat + chain := PreRouting + rule := fmt.Sprintf("-p ARP -i %s --arp-op Reply -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT", + interfaceName) + + return runEbCmd(table, action, chain, rule) +} + +// SetVepaMode sets the VEPA mode for a bridge and its ports. +func SetVepaMode(bridgeName string, downstreamIfNamePrefix string, upstreamMacAddress string, action string) error { + table := Nat + chain := PreRouting + + if !strings.HasPrefix(bridgeName, downstreamIfNamePrefix) { + rule := fmt.Sprintf("-i %s -j dnat --to-dst %s --dnat-target ACCEPT", bridgeName, upstreamMacAddress) + + err := runEbCmd(table, action, chain, rule) + if err != nil { + return err + } + } + + rule2 := fmt.Sprintf("-i %s+ -j dnat --to-dst %s --dnat-target ACCEPT", + downstreamIfNamePrefix, upstreamMacAddress) + + return runEbCmd(table, action, chain, rule2) +} + +// SetDnatForIPAddress sets a MAC DNAT rule for an IP address. +func SetDnatForIPAddress(interfaceName string, ipAddress net.IP, macAddress net.HardwareAddr, action string) error { + table := Nat + chain := PreRouting + rule := fmt.Sprintf("-p IPv4 -i %s --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT", + interfaceName, ipAddress.String(), macAddress.String()) + + return runEbCmd(table, action, chain, rule) +} + +// SetEbRule sets any given eb rule +func SetEbRule(table, action, chain, rule string) error { + + return runEbCmd(table, action, chain, rule) } // GetEbtableRules gets EB rules for a table and chain. @@ -114,50 +151,10 @@ func EbTableRuleExists(tableName, chainName, matchSet string) (bool, error) { return false, nil } -// SetDnatForArpReplies sets a MAC DNAT rule for ARP replies received on an interface. -func SetDnatForArpReplies(interfaceName string, action string) error { - command := fmt.Sprintf( - "ebtables -t nat %s PREROUTING -p ARP -i %s --arp-op Reply -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT", - action, interfaceName) - - return executeShellCommand(command) -} - -// SetVepaMode sets the VEPA mode for a bridge and its ports. -func SetVepaMode(bridgeName string, downstreamIfNamePrefix string, upstreamMacAddress string, action string) error { - if !strings.HasPrefix(bridgeName, downstreamIfNamePrefix) { - command := fmt.Sprintf( - "ebtables -t nat %s PREROUTING -i %s -j dnat --to-dst %s --dnat-target ACCEPT", - action, bridgeName, upstreamMacAddress) - - err := executeShellCommand(command) - if err != nil { - return err - } - } - - command := fmt.Sprintf( - "ebtables -t nat %s PREROUTING -i %s+ -j dnat --to-dst %s --dnat-target ACCEPT", - action, downstreamIfNamePrefix, upstreamMacAddress) - - return executeShellCommand(command) -} - -// SetDnatForIPAddress sets a MAC DNAT rule for an IP address. -func SetDnatForIPAddress(interfaceName string, ipAddress net.IP, macAddress net.HardwareAddr, action string) error { - command := fmt.Sprintf( - "ebtables -t nat %s PREROUTING -p IPv4 -i %s --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT", - action, interfaceName, ipAddress.String(), macAddress.String()) - - return executeShellCommand(command) -} +// runEbCmd runs an EB rule command. +func runEbCmd(table, action, chain, rule string) error { + command := fmt.Sprintf("ebtables -t %s %s %s %s", table, action, chain, rule) + _, err := platform.ExecuteCommand(command) -func executeShellCommand(command string) error { - log.Debugf("[ebtables] %s", command) - cmd := exec.Command("sh", "-c", command) - err := cmd.Start() - if err != nil { - return err - } - return cmd.Wait() + return err } diff --git a/move-to-cluster.sh b/move-to-cluster.sh new file mode 100755 index 0000000000..859a5752c8 --- /dev/null +++ b/move-to-cluster.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +scp output/linux_amd64/cni/*.tgz aks-test-cluster:~/ +scp output/linux_amd64/cnms/azure-cnms aks-test-cluster:~/ diff --git a/network/manager.go b/network/manager.go index 98bcbf5ace..6cfd1bdbec 100644 --- a/network/manager.go +++ b/network/manager.go @@ -7,6 +7,7 @@ import ( "sync" "time" + cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" @@ -67,6 +68,7 @@ type NetworkManager interface { DetachEndpoint(networkId string, endpointId string) error UpdateEndpoint(networkId string, existingEpInfo *EndpointInfo, targetEpInfo *EndpointInfo) error GetNumberOfEndpoints(ifName string, networkId string) int + SetupNetworkUsingState(networkMonitor *cnms.NetworkMonitor) error } // Creates a new network manager. @@ -105,6 +107,9 @@ func (nm *networkManager) restore() error { // Ignore the persisted state if it is older than the last reboot time. // Read any persisted state. + nm.Lock() + defer nm.Unlock() + err := nm.store.Read(storeKey, nm) if err != nil { if err == store.ErrKeyNotFound { @@ -487,3 +492,8 @@ func (nm *networkManager) GetNumberOfEndpoints(ifName string, networkId string) return 0 } + +func (nm *networkManager) SetupNetworkUsingState(networkMonitor *cnms.NetworkMonitor) error { + log.Printf("SetupNetworkUsingState function called") + return nm.monitorNetworkState(networkMonitor) +} diff --git a/network/monitor_linux.go b/network/monitor_linux.go new file mode 100644 index 0000000000..d79293d03e --- /dev/null +++ b/network/monitor_linux.go @@ -0,0 +1,63 @@ +package network + +import ( + "fmt" + + cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" + "github.com/Azure/azure-container-networking/ebtables" + "github.com/Azure/azure-container-networking/log" +) + +func (nm *networkManager) monitorNetworkState(networkMonitor *cnms.NetworkMonitor) error { + log.Printf("monitorNetworkState function called\n") + currentEbtableRulesMap, err := cnms.GetEbTableRulesInMap() + + if err != nil { + log.Printf("GetEbTableRulesInMap failed with error %v", err) + return err + } + + currentStateRulesMap := nm.AddStateRulesToMap() + + log.Printf("CurrentEbtableRulesMap: \n%v\n", currentEbtableRulesMap) + log.Printf("currentStateRulesMap: \n%v\n", currentStateRulesMap) + + networkMonitor.CreateRequiredL2Rules(currentEbtableRulesMap, currentStateRulesMap) + + log.Printf("CurrentEbtableRulesMap after CreateRequiredl2rules: \n%v\n", currentEbtableRulesMap) + log.Printf("currentStateRulesMap after CreateRequiredL2Rules: \n%v\n", currentStateRulesMap) + + networkMonitor.RemoveInvalidL2Rules(currentEbtableRulesMap, currentStateRulesMap) + + log.Printf("CurrentEbtableRulesMap after RemoveInvalid: \n%v\n", currentEbtableRulesMap) + log.Printf("currentStateRulesMap after RemoveInvalid: \n%v\n", currentStateRulesMap) + + return nil +} + +func (nm *networkManager) AddStateRulesToMap() map[string]string { + log.Printf("AddStateRulesToMap funciton called\n") + rulesMap := make(map[string]string) + + for _, extIf := range nm.ExternalInterfaces { + arpDnatKey := fmt.Sprintf("-p ARP -i %s --arp-op Reply -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT", extIf.Name) + rulesMap[arpDnatKey] = ebtables.PreRouting + + snatKey := fmt.Sprintf("-s Unicast -o %s -j snat --to-src %s --snat-arp --snat-target ACCEPT", extIf.Name, extIf.MacAddress.String()) + rulesMap[snatKey] = ebtables.PostRouting + + for _, nw := range extIf.Networks { + for _, ep := range nw.Endpoints { + for _, ipAddr := range ep.IPAddresses { + arpReplyKey := fmt.Sprintf("-p ARP --arp-op Request --arp-ip-dst %s -j arpreply --arpreply-mac %s", ipAddr.IP.String(), ep.MacAddress.String()) + rulesMap[arpReplyKey] = ebtables.PreRouting + + dnatMacKey := fmt.Sprintf("-p IPv4 -i %s --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT", extIf.Name, ipAddr.IP.String(), ep.MacAddress.String()) + rulesMap[dnatMacKey] = ebtables.PreRouting + } + } + } + } + + return rulesMap +} diff --git a/network/monitor_windows.go b/network/monitor_windows.go new file mode 100644 index 0000000000..a3b13d62d8 --- /dev/null +++ b/network/monitor_windows.go @@ -0,0 +1,3 @@ +func (nm *networkManager) monitorNetworkState() error { + return nil +} \ No newline at end of file diff --git a/networkmonitor/Dockerfile b/networkmonitor/Dockerfile new file mode 100644 index 0000000000..21a2c1adab --- /dev/null +++ b/networkmonitor/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:latest +RUN apt -y update +RUN apt install -y ebtables +RUN apt install -y net-tools +COPY networkmonitor /usr/bin/networkmonitor +CMD ["/usr/bin/networkmonitor"] \ No newline at end of file diff --git a/networkmonitor/main.go b/networkmonitor/main.go new file mode 100644 index 0000000000..6c8529b314 --- /dev/null +++ b/networkmonitor/main.go @@ -0,0 +1,153 @@ +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package main + +import ( + "fmt" + "os" + "time" + + cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" + acn "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network" + "github.com/Azure/azure-container-networking/platform" + "github.com/Azure/azure-container-networking/store" +) + +const ( + // Service name. + name = "azure-cnimonitor" + pluginName = "azure-vnet" + DEFAULT_TIMEOUT_IN_SECS = "10" +) + +// Version is populated by make during build. +var version string + +// Command line arguments for CNM plugin. +var args = acn.ArgumentList{ + { + Name: acn.OptLogLevel, + Shorthand: acn.OptLogLevelAlias, + Description: "Set the logging level", + Type: "int", + DefaultValue: acn.OptLogLevelInfo, + ValueMap: map[string]interface{}{ + acn.OptLogLevelInfo: log.LevelInfo, + acn.OptLogLevelDebug: log.LevelDebug, + }, + }, + { + Name: acn.OptLogTarget, + Shorthand: acn.OptLogTargetAlias, + Description: "Set the logging target", + Type: "int", + DefaultValue: acn.OptLogTargetFile, + ValueMap: map[string]interface{}{ + acn.OptLogTargetSyslog: log.TargetSyslog, + acn.OptLogTargetStderr: log.TargetStderr, + acn.OptLogTargetFile: log.TargetLogfile, + }, + }, + { + Name: acn.OptLogLocation, + Shorthand: acn.OptLogLocationAlias, + Description: "Set the directory location where logs will be saved", + Type: "string", + DefaultValue: "", + }, + { + Name: acn.OptIntervalTime, + Shorthand: acn.OptIntervalTimeAlias, + Description: "Periodic Interval Time", + Type: "int", + DefaultValue: DEFAULT_TIMEOUT_IN_SECS, + }, + { + Name: acn.OptVersion, + Shorthand: acn.OptVersionAlias, + Description: "Print version information", + Type: "bool", + DefaultValue: false, + }, +} + +// Prints description and version information. +func printVersion() { + fmt.Printf("Azure Container Network Service\n") + fmt.Printf("Version %v\n", version) +} + +// Main is the entry point for CNS. +func main() { + // Initialize and parse command line arguments. + acn.ParseArgs(&args, printVersion) + + logLevel := acn.GetArg(acn.OptLogLevel).(int) + logTarget := acn.GetArg(acn.OptLogTarget).(int) + logDirectory := acn.GetArg(acn.OptLogLocation).(string) + timeout := acn.GetArg(acn.OptIntervalTime).(int) + vers := acn.GetArg(acn.OptVersion).(bool) + + if vers { + printVersion() + os.Exit(0) + } + + // Initialize CNS. + var config acn.PluginConfig + config.Version = version + + // Create a channel to receive unhandled errors from CNS. + config.ErrChan = make(chan error, 1) + + var err error + // Create logging provider. + log.SetName(name) + log.SetLevel(logLevel) + if err := log.SetTargetLogDirectory(logTarget, logDirectory); err != nil { + fmt.Printf("[netmon] Failed to configure logging: %v\n", err) + return + } + + // Log platform information. + log.Printf("Running on %v", platform.GetOSInfo()) + + netMonitor := &cnms.NetworkMonitor{ + AddRulesToBeValidated: make(map[string]int), + DeleteRulesToBeValidated: make(map[string]int), + } + + for true { + config.Store, err = store.NewJsonFileStore(platform.CNIRuntimePath + pluginName + ".json") + if err != nil { + fmt.Printf("Failed to create store: %v\n", err) + return + } + + nm, err := network.NewNetworkManager() + if err != nil { + log.Printf("Failed while creating network manager") + return + } + + if err := nm.Initialize(&config); err != nil { + log.Printf("Failed while initializing network manager %+v", err) + } + + log.Printf("network manager:%+v", nm) + + if err := nm.SetupNetworkUsingState(netMonitor); err != nil { + log.Printf("Failed while calling SetupNetworkUsingState with error %v", err) + return + } + + log.Printf("Going to sleep for %v seconds", timeout) + time.Sleep(time.Duration(timeout) * time.Second) + nm = nil + } + + log.Close() +} From b5950f9bd3806f802a6cdba813d9a3293a247475 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Thu, 12 Mar 2020 10:48:03 -0700 Subject: [PATCH 2/6] Removed debugging statements and tested again --- cni/network/network_linux.go | 2 -- cnms/cnmspackage/monitor2rules_linux.go | 43 ++++++++----------------- cnms/service/networkmonitor.go | 17 +++++----- move-to-cluster.sh | 4 --- network/monitor_linux.go | 14 ++------ networkmonitor/main.go | 16 ++++----- 6 files changed, 31 insertions(+), 65 deletions(-) delete mode 100755 move-to-cluster.sh diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index a375568457..0ee0b8f7c0 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -90,8 +90,6 @@ func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { } func startMonitorIfNotRunning(nwCfg *cni.NetworkConfig) error { - log.Printf("startMonitorIfNotRunning function called") - if nwCfg != nil { netmon := nwCfg.NetworkMonitor diff --git a/cnms/cnmspackage/monitor2rules_linux.go b/cnms/cnmspackage/monitor2rules_linux.go index 1903b5c144..0ecfb8b315 100644 --- a/cnms/cnmspackage/monitor2rules_linux.go +++ b/cnms/cnmspackage/monitor2rules_linux.go @@ -6,10 +6,10 @@ import ( ) func (networkMonitor *NetworkMonitor) deleteRulesNotExistInMap(chainRules map[string]string, stateRules map[string]string) { - log.Printf("deleteRulesNotExistInMap function called\n") - table := "nat" + + table := ebtables.Nat action := ebtables.Delete - //Here I think + for rule, chain := range chainRules { if _, ok := stateRules[rule]; !ok { if itr, ok := networkMonitor.DeleteRulesToBeValidated[rule]; ok && itr > 0 { @@ -28,8 +28,8 @@ func (networkMonitor *NetworkMonitor) deleteRulesNotExistInMap(chainRules map[st } func deleteRulesExistInMap(originalChainRules map[string]string, stateRules map[string]string) { - log.Printf("deleteRulesExistinMap funciton called\n") - table := "nat" + + table := ebtables.Nat action := ebtables.Delete for rule, chain := range originalChainRules { @@ -45,9 +45,8 @@ func deleteRulesExistInMap(originalChainRules map[string]string, stateRules map[ func (networkMonitor *NetworkMonitor) addRulesNotExistInMap( stateRules map[string]string, chainRules map[string]string) { - log.Printf("addRulesNotExistInMap funciton called \n") - table := "nat" + table := ebtables.Nat action := ebtables.Append for rule, chain := range stateRules { @@ -70,12 +69,11 @@ func (networkMonitor *NetworkMonitor) addRulesNotExistInMap( func (networkMonitor *NetworkMonitor) CreateRequiredL2Rules( currentEbtableRulesMap map[string]string, currentStateRulesMap map[string]string) error { - log.Printf("CreateRequiredL2Rules function called\n") for rule := range networkMonitor.AddRulesToBeValidated { - log.Printf("Rule in AddRulesToBeValidated %v", rule) + log.Printf("[monitor] Rule in AddRulesToBeValidated %v", rule) if _, ok := currentStateRulesMap[rule]; !ok { - log.Printf("Deleting Rule") + log.Printf("[monitor] Deleting Rule from AddRulesToBeValidated %v", rule) delete(networkMonitor.AddRulesToBeValidated, rule) } } @@ -88,36 +86,23 @@ func (networkMonitor *NetworkMonitor) CreateRequiredL2Rules( func (networkMonitor *NetworkMonitor) RemoveInvalidL2Rules( currentEbtableRulesMap map[string]string, currentStateRulesMap map[string]string) error { - log.Printf("RemoveInvalidL2Rules function called\n") for rule := range networkMonitor.DeleteRulesToBeValidated { - log.Printf("Checking DeleteRulesToBeValidated rule: %v", rule) + log.Printf("[monitor] Checking DeleteRulesToBeValidated rule: %v", rule) if _, ok := currentEbtableRulesMap[rule]; !ok { - log.Printf("DeleteRulesToBeValidated deleting rule: %v", rule) + log.Printf("[monitor] DeleteRulesToBeValidated deleting rule: %v", rule) delete(networkMonitor.DeleteRulesToBeValidated, rule) } } - // originalChainRules := make(map[string]string) - - // if err := generateL2RulesMap(originalChainRules, ebtables.PreRouting); err != nil { - // return err - // } - - // if err := generateL2RulesMap(originalChainRules, ebtables.PostRouting); err != nil { - // return err - // } - - // deleteRulesExistInMap(originalChainRules, currentEbtableRulesMap) - networkMonitor.deleteRulesNotExistInMap(currentEbtableRulesMap, currentStateRulesMap) return nil } func generateL2RulesMap(currentEbtableRulesMap map[string]string, chainName string) error { - log.Printf("generateL2RulesMap funciton called\n") - table := "nat" + + table := ebtables.Nat rules, err := ebtables.GetEbtableRules(table, chainName) if err != nil { @@ -126,8 +111,6 @@ func generateL2RulesMap(currentEbtableRulesMap map[string]string, chainName stri return err } - log.Printf("[monitor] Rules count : %v", len(rules)) - for _, rule := range rules { log.Printf("[monitor] Adding rule %s mapped to chainName %s.", rule, chainName) currentEbtableRulesMap[rule] = chainName @@ -137,7 +120,7 @@ func generateL2RulesMap(currentEbtableRulesMap map[string]string, chainName stri } func GetEbTableRulesInMap() (map[string]string, error) { - log.Printf("GetEbtableRulesInMap funciton called \n") + currentEbtableRulesMap := make(map[string]string) if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PreRouting); err != nil { diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go index 9ce72d1d5b..7736dfc279 100644 --- a/cnms/service/networkmonitor.go +++ b/cnms/service/networkmonitor.go @@ -21,7 +21,6 @@ const ( name = "azure-cnimonitor" pluginName = "azure-vnet" DEFAULT_TIMEOUT_IN_SECS = "10" - DefaultMode = "AzureChain" ) // Version is populated by make during build. @@ -109,12 +108,12 @@ func main() { log.SetName(name) log.SetLevel(logLevel) if err := log.SetTargetLogDirectory(logTarget, logDirectory); err != nil { - fmt.Printf("[netmon] Failed to configure logging: %v\n", err) + fmt.Printf("[monitor] Failed to configure logging: %v\n", err) return } // Log platform information. - log.Printf("[netmon] Running on %v", platform.GetOSInfo()) + log.Printf("[monitor] Running on %v", platform.GetOSInfo()) netMonitor := &cnms.NetworkMonitor{ AddRulesToBeValidated: make(map[string]int), @@ -124,27 +123,27 @@ func main() { for true { config.Store, err = store.NewJsonFileStore(platform.CNIRuntimePath + pluginName + ".json") if err != nil { - fmt.Printf("[netmon] Failed to create store: %v\n", err) + fmt.Printf("[monitor] Failed to create store: %v\n", err) return } nm, err := network.NewNetworkManager() if err != nil { - log.Printf("[netmon] Failed while creating network manager") + log.Printf("[monitor] Failed while creating network manager") return } if err := nm.Initialize(&config); err != nil { - log.Printf("[netmon] Failed while initializing network manager %+v", err) + log.Printf("[monitor] Failed while initializing network manager %+v", err) } - log.Printf("[netmon] network manager:%+v", nm) + log.Printf("[monitor] network manager:%+v", nm) if err := nm.SetupNetworkUsingState(netMonitor); err != nil { - log.Printf("[netmon] Failed while calling SetupNetworkUsingState with error %v", err) + log.Printf("[monitor] Failed while calling SetupNetworkUsingState with error %v", err) } - log.Printf("[netmon] Going to sleep for %v seconds", timeout) + log.Printf("[monitor] Going to sleep for %v seconds", timeout) time.Sleep(time.Duration(timeout) * time.Second) nm = nil } diff --git a/move-to-cluster.sh b/move-to-cluster.sh deleted file mode 100755 index 859a5752c8..0000000000 --- a/move-to-cluster.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -scp output/linux_amd64/cni/*.tgz aks-test-cluster:~/ -scp output/linux_amd64/cnms/azure-cnms aks-test-cluster:~/ diff --git a/network/monitor_linux.go b/network/monitor_linux.go index d79293d03e..dcde1348ff 100644 --- a/network/monitor_linux.go +++ b/network/monitor_linux.go @@ -9,7 +9,7 @@ import ( ) func (nm *networkManager) monitorNetworkState(networkMonitor *cnms.NetworkMonitor) error { - log.Printf("monitorNetworkState function called\n") + currentEbtableRulesMap, err := cnms.GetEbTableRulesInMap() if err != nil { @@ -19,24 +19,14 @@ func (nm *networkManager) monitorNetworkState(networkMonitor *cnms.NetworkMonito currentStateRulesMap := nm.AddStateRulesToMap() - log.Printf("CurrentEbtableRulesMap: \n%v\n", currentEbtableRulesMap) - log.Printf("currentStateRulesMap: \n%v\n", currentStateRulesMap) - networkMonitor.CreateRequiredL2Rules(currentEbtableRulesMap, currentStateRulesMap) - - log.Printf("CurrentEbtableRulesMap after CreateRequiredl2rules: \n%v\n", currentEbtableRulesMap) - log.Printf("currentStateRulesMap after CreateRequiredL2Rules: \n%v\n", currentStateRulesMap) - networkMonitor.RemoveInvalidL2Rules(currentEbtableRulesMap, currentStateRulesMap) - log.Printf("CurrentEbtableRulesMap after RemoveInvalid: \n%v\n", currentEbtableRulesMap) - log.Printf("currentStateRulesMap after RemoveInvalid: \n%v\n", currentStateRulesMap) - return nil } func (nm *networkManager) AddStateRulesToMap() map[string]string { - log.Printf("AddStateRulesToMap funciton called\n") + rulesMap := make(map[string]string) for _, extIf := range nm.ExternalInterfaces { diff --git a/networkmonitor/main.go b/networkmonitor/main.go index 6c8529b314..822c1d3301 100644 --- a/networkmonitor/main.go +++ b/networkmonitor/main.go @@ -108,12 +108,12 @@ func main() { log.SetName(name) log.SetLevel(logLevel) if err := log.SetTargetLogDirectory(logTarget, logDirectory); err != nil { - fmt.Printf("[netmon] Failed to configure logging: %v\n", err) + fmt.Printf("[monitor] Failed to configure logging: %v\n", err) return } // Log platform information. - log.Printf("Running on %v", platform.GetOSInfo()) + log.Printf("[monitor] Running on %v", platform.GetOSInfo()) netMonitor := &cnms.NetworkMonitor{ AddRulesToBeValidated: make(map[string]int), @@ -123,28 +123,28 @@ func main() { for true { config.Store, err = store.NewJsonFileStore(platform.CNIRuntimePath + pluginName + ".json") if err != nil { - fmt.Printf("Failed to create store: %v\n", err) + fmt.Printf("[monitor] Failed to create store: %v\n", err) return } nm, err := network.NewNetworkManager() if err != nil { - log.Printf("Failed while creating network manager") + log.Printf("[monitor] Failed while creating network manager") return } if err := nm.Initialize(&config); err != nil { - log.Printf("Failed while initializing network manager %+v", err) + log.Printf("[monitor] Failed while initializing network manager %+v", err) } - log.Printf("network manager:%+v", nm) + log.Printf("[monitor] network manager:%+v", nm) if err := nm.SetupNetworkUsingState(netMonitor); err != nil { - log.Printf("Failed while calling SetupNetworkUsingState with error %v", err) + log.Printf("[monitor] Failed while calling SetupNetworkUsingState with error %v", err) return } - log.Printf("Going to sleep for %v seconds", timeout) + log.Printf("[monitor] Going to sleep for %v seconds", timeout) time.Sleep(time.Duration(timeout) * time.Second) nm = nil } From 49292609bac645f0d4f78fd11f22033745981634 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Thu, 12 Mar 2020 13:51:45 -0700 Subject: [PATCH 3/6] Removed more debugging statements and unused method --- cnms/cnmspackage/monitor2rules_linux.go | 20 -------------------- network/manager.go | 1 - 2 files changed, 21 deletions(-) diff --git a/cnms/cnmspackage/monitor2rules_linux.go b/cnms/cnmspackage/monitor2rules_linux.go index 0ecfb8b315..24807a27aa 100644 --- a/cnms/cnmspackage/monitor2rules_linux.go +++ b/cnms/cnmspackage/monitor2rules_linux.go @@ -27,21 +27,6 @@ func (networkMonitor *NetworkMonitor) deleteRulesNotExistInMap(chainRules map[st } } -func deleteRulesExistInMap(originalChainRules map[string]string, stateRules map[string]string) { - - table := ebtables.Nat - action := ebtables.Delete - - for rule, chain := range originalChainRules { - if _, ok := stateRules[rule]; ok { - log.Printf("[monitor] Deleting Ebtable rule which existed in map %v", rule) - if err := ebtables.SetEbRule(table, action, chain, rule); err != nil { - log.Printf("[monitor] Error while deleting ebtable rule %v", err) - } - } - } -} - func (networkMonitor *NetworkMonitor) addRulesNotExistInMap( stateRules map[string]string, chainRules map[string]string) { @@ -71,9 +56,7 @@ func (networkMonitor *NetworkMonitor) CreateRequiredL2Rules( currentStateRulesMap map[string]string) error { for rule := range networkMonitor.AddRulesToBeValidated { - log.Printf("[monitor] Rule in AddRulesToBeValidated %v", rule) if _, ok := currentStateRulesMap[rule]; !ok { - log.Printf("[monitor] Deleting Rule from AddRulesToBeValidated %v", rule) delete(networkMonitor.AddRulesToBeValidated, rule) } } @@ -88,9 +71,7 @@ func (networkMonitor *NetworkMonitor) RemoveInvalidL2Rules( currentStateRulesMap map[string]string) error { for rule := range networkMonitor.DeleteRulesToBeValidated { - log.Printf("[monitor] Checking DeleteRulesToBeValidated rule: %v", rule) if _, ok := currentEbtableRulesMap[rule]; !ok { - log.Printf("[monitor] DeleteRulesToBeValidated deleting rule: %v", rule) delete(networkMonitor.DeleteRulesToBeValidated, rule) } } @@ -112,7 +93,6 @@ func generateL2RulesMap(currentEbtableRulesMap map[string]string, chainName stri } for _, rule := range rules { - log.Printf("[monitor] Adding rule %s mapped to chainName %s.", rule, chainName) currentEbtableRulesMap[rule] = chainName } diff --git a/network/manager.go b/network/manager.go index 6cfd1bdbec..18252be530 100644 --- a/network/manager.go +++ b/network/manager.go @@ -494,6 +494,5 @@ func (nm *networkManager) GetNumberOfEndpoints(ifName string, networkId string) } func (nm *networkManager) SetupNetworkUsingState(networkMonitor *cnms.NetworkMonitor) error { - log.Printf("SetupNetworkUsingState function called") return nm.monitorNetworkState(networkMonitor) } From fa90195ceeeacfe46dd9f45588bc7ac9549eb1fa Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Fri, 13 Mar 2020 11:29:48 -0700 Subject: [PATCH 4/6] Made changes suggested by Tamilmani, tested again. --- cni/azure-linux.conflist | 3 - cni/netconfig.go | 5 - cni/network/network.go | 4 - cni/network/network_linux.go | 40 ------- cni/network/network_windows.go | 3 - cnms/Dockerfile | 1 + cnms/cnmspackage/monitor2rules_linux.go | 15 ++- cnms/service/networkmonitor.go | 2 - ebtables/ebtables.go | 5 +- network/monitor_linux.go | 7 +- networkmonitor/Dockerfile | 6 - networkmonitor/main.go | 153 ------------------------ 12 files changed, 12 insertions(+), 232 deletions(-) delete mode 100644 networkmonitor/Dockerfile delete mode 100644 networkmonitor/main.go diff --git a/cni/azure-linux.conflist b/cni/azure-linux.conflist index 4793e49d1c..fad44fb448 100644 --- a/cni/azure-linux.conflist +++ b/cni/azure-linux.conflist @@ -9,9 +9,6 @@ "ipsToRouteViaHost":["169.254.20.10"], "ipam":{ "type":"azure-vnet-ipam" - }, - "networkMonitor":{ - "name":"azure-cnms" } }, { diff --git a/cni/netconfig.go b/cni/netconfig.go index 5d3142b355..54a1c13995 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -67,11 +67,6 @@ type NetworkConfig struct { Address string `json:"ipAddress,omitempty"` QueryInterval string `json:"queryInterval,omitempty"` } - NetworkMonitor struct { - Name string `json:"name"` - Interval int `json:"interval,omitempty"` - Disable bool `json:"disable,omitempty"` - } DNS cniTypes.DNS `json:"dns"` RuntimeConfig RuntimeConfig `json:"runtimeConfig"` AdditionalArgs []KVPair diff --git a/cni/network/network.go b/cni/network/network.go index 5400cb3282..35a7f0881c 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -263,10 +263,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Read network configuration %+v.", nwCfg) - if err := startMonitorIfNotRunning(nwCfg); err != nil { - log.Printf("Starting network monitor failed with %v", err) - } - // Temporary if block to determing whether we disable SNAT on host (for multi-tenant scenario only) if nwCfg.MultiTenancy { if enableSnatForDns, nwCfg.EnableSnatOnHost, err = determineSnat(); err != nil { diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 0ee0b8f7c0..81c8e31eb4 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -1,9 +1,7 @@ package network import ( - "fmt" "net" - "os/exec" "strconv" "github.com/Azure/azure-container-networking/cni" @@ -11,7 +9,6 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/policy" - "github.com/Azure/azure-container-networking/platform" cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypes "github.com/containernetworking/cni/pkg/types" @@ -89,43 +86,6 @@ func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { } } -func startMonitorIfNotRunning(nwCfg *cni.NetworkConfig) error { - if nwCfg != nil { - - netmon := nwCfg.NetworkMonitor - if netmon.Name == "" { - return fmt.Errorf("NetworkMonitor is not set in conflist") - } - - if netmon.Disable == true { - log.Printf("Network Monitor is not enabled") - return nil - } - - cmd := fmt.Sprintf("pgrep -x %v", netmon.Name) - _, err := platform.ExecuteCommand(cmd) - if err == nil { - log.Printf("Azure Network monitor is already running") - } - - cmd = fmt.Sprintf(binPath+"/%v", netmon.Name) - - if netmon.Interval != 0 { - cmd = fmt.Sprintf("%v -it %v", cmd, netmon.Interval) - } - - startCmd := exec.Command("sh", "-c", cmd) - if err := startCmd.Start(); err != nil { - log.Printf("startcmd failed with error %v", err) - return err - } - - log.Printf("Azure Network Monitor started") - } - - return nil -} - func setupInfraVnetRoutingForMultitenancy( nwCfg *cni.NetworkConfig, azIpamResult *cniTypesCurr.Result, diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 481437b0e6..615c871496 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -118,9 +118,6 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { } -func startMonitorIfNotRunning(nwCfg *cni.NetworkConfig) error { -} - func updateSubnetPrefix(cnsNwConfig *cns.GetNetworkContainerResponse, subnetPrefix *net.IPNet) error { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { ipconfig := cnsNwConfig.IPConfiguration diff --git a/cnms/Dockerfile b/cnms/Dockerfile index 21a2c1adab..1c3f0da745 100644 --- a/cnms/Dockerfile +++ b/cnms/Dockerfile @@ -1,5 +1,6 @@ FROM ubuntu:latest RUN apt -y update +RUN apt-get -y upgrade RUN apt install -y ebtables RUN apt install -y net-tools COPY networkmonitor /usr/bin/networkmonitor diff --git a/cnms/cnmspackage/monitor2rules_linux.go b/cnms/cnmspackage/monitor2rules_linux.go index 24807a27aa..9d2440bdfa 100644 --- a/cnms/cnmspackage/monitor2rules_linux.go +++ b/cnms/cnmspackage/monitor2rules_linux.go @@ -5,6 +5,7 @@ import ( "github.com/Azure/azure-container-networking/log" ) +// deleteRulesNotExistInMap deletes rules from nat Ebtable if rule was not in stateRules after a certain number of iterations. func (networkMonitor *NetworkMonitor) deleteRulesNotExistInMap(chainRules map[string]string, stateRules map[string]string) { table := ebtables.Nat @@ -27,6 +28,7 @@ func (networkMonitor *NetworkMonitor) deleteRulesNotExistInMap(chainRules map[st } } +// addRulesNotExistInMap adds rules to nat Ebtable if rule was in stateRules and not in current chain rules after a certain number of iterations. func (networkMonitor *NetworkMonitor) addRulesNotExistInMap( stateRules map[string]string, chainRules map[string]string) { @@ -37,7 +39,7 @@ func (networkMonitor *NetworkMonitor) addRulesNotExistInMap( for rule, chain := range stateRules { if _, ok := chainRules[rule]; !ok { if itr, ok := networkMonitor.AddRulesToBeValidated[rule]; ok && itr > 0 { - log.Printf("[monitor] Adding Ebtable rule as it didn't exist in state for %d iterations chain %v rule %v", itr, chain, rule) + log.Printf("[monitor] Adding Ebtable rule as it existed in state rules but not in current chain rules for %d iterations chain %v rule %v", itr, chain, rule) if err := ebtables.SetEbRule(table, action, chain, rule); err != nil { log.Printf("[monitor] Error while adding ebtable rule %v", err) } @@ -51,6 +53,7 @@ func (networkMonitor *NetworkMonitor) addRulesNotExistInMap( } } +// CreateRequiredL2Rules finds the rules that should be in nat ebtable based on state. func (networkMonitor *NetworkMonitor) CreateRequiredL2Rules( currentEbtableRulesMap map[string]string, currentStateRulesMap map[string]string) error { @@ -60,12 +63,12 @@ func (networkMonitor *NetworkMonitor) CreateRequiredL2Rules( delete(networkMonitor.AddRulesToBeValidated, rule) } } - networkMonitor.addRulesNotExistInMap(currentStateRulesMap, currentEbtableRulesMap) return nil } +// RemoveInvalidL2Rules removes rules that should not be in nat ebtable based on state. func (networkMonitor *NetworkMonitor) RemoveInvalidL2Rules( currentEbtableRulesMap map[string]string, currentStateRulesMap map[string]string) error { @@ -75,17 +78,15 @@ func (networkMonitor *NetworkMonitor) RemoveInvalidL2Rules( delete(networkMonitor.DeleteRulesToBeValidated, rule) } } - networkMonitor.deleteRulesNotExistInMap(currentEbtableRulesMap, currentStateRulesMap) return nil } +// generateL2RulesMap gets rules from chainName and puts them in currentEbtableRulesMap. func generateL2RulesMap(currentEbtableRulesMap map[string]string, chainName string) error { - table := ebtables.Nat rules, err := ebtables.GetEbtableRules(table, chainName) - if err != nil { log.Printf("[monitor] Error while getting rules list from table %v chain %v. Error: %v", table, chainName, err) @@ -99,14 +100,12 @@ func generateL2RulesMap(currentEbtableRulesMap map[string]string, chainName stri return nil } +// GetEbTableRulesInMap gathers prerouting and postrouting rules into a map. func GetEbTableRulesInMap() (map[string]string, error) { - currentEbtableRulesMap := make(map[string]string) - if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PreRouting); err != nil { return nil, err } - if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PostRouting); err != nil { return nil, err } diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go index 7736dfc279..a75bae2b94 100644 --- a/cnms/service/networkmonitor.go +++ b/cnms/service/networkmonitor.go @@ -84,13 +84,11 @@ func printVersion() { func main() { // Initialize and parse command line arguments. acn.ParseArgs(&args, printVersion) - logLevel := acn.GetArg(acn.OptLogLevel).(int) logTarget := acn.GetArg(acn.OptLogTarget).(int) logDirectory := acn.GetArg(acn.OptLogLocation).(string) timeout := acn.GetArg(acn.OptIntervalTime).(int) vers := acn.GetArg(acn.OptVersion).(bool) - if vers { printVersion() os.Exit(0) diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go index d241d11719..3dceb5f195 100644 --- a/ebtables/ebtables.go +++ b/ebtables/ebtables.go @@ -15,10 +15,10 @@ const ( // Ebtables actions. Append = "-A" Delete = "-D" - // Ebtables + // Ebtables tables. Nat = "nat" Broute = "broute" - // Ebtable chains + // Ebtable chains. PreRouting = "PREROUTING" PostRouting = "POSTROUTING" Brouting = "BROUTING" @@ -95,7 +95,6 @@ func SetDnatForIPAddress(interfaceName string, ipAddress net.IP, macAddress net. // SetEbRule sets any given eb rule func SetEbRule(table, action, chain, rule string) error { - return runEbCmd(table, action, chain, rule) } diff --git a/network/monitor_linux.go b/network/monitor_linux.go index dcde1348ff..4c85bb99c6 100644 --- a/network/monitor_linux.go +++ b/network/monitor_linux.go @@ -8,25 +8,22 @@ import ( "github.com/Azure/azure-container-networking/log" ) +// monitorNetworkState compares current ebtable nat rules with state rules and matches state. func (nm *networkManager) monitorNetworkState(networkMonitor *cnms.NetworkMonitor) error { - currentEbtableRulesMap, err := cnms.GetEbTableRulesInMap() - if err != nil { log.Printf("GetEbTableRulesInMap failed with error %v", err) return err } - currentStateRulesMap := nm.AddStateRulesToMap() - networkMonitor.CreateRequiredL2Rules(currentEbtableRulesMap, currentStateRulesMap) networkMonitor.RemoveInvalidL2Rules(currentEbtableRulesMap, currentStateRulesMap) return nil } +// AddStateRulesToMap adds rules to state based off network manager settings. func (nm *networkManager) AddStateRulesToMap() map[string]string { - rulesMap := make(map[string]string) for _, extIf := range nm.ExternalInterfaces { diff --git a/networkmonitor/Dockerfile b/networkmonitor/Dockerfile deleted file mode 100644 index 21a2c1adab..0000000000 --- a/networkmonitor/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM ubuntu:latest -RUN apt -y update -RUN apt install -y ebtables -RUN apt install -y net-tools -COPY networkmonitor /usr/bin/networkmonitor -CMD ["/usr/bin/networkmonitor"] \ No newline at end of file diff --git a/networkmonitor/main.go b/networkmonitor/main.go deleted file mode 100644 index 822c1d3301..0000000000 --- a/networkmonitor/main.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2017 Microsoft. All rights reserved. -// MIT License - -package main - -import ( - "fmt" - "os" - "time" - - cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" - acn "github.com/Azure/azure-container-networking/common" - "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/network" - "github.com/Azure/azure-container-networking/platform" - "github.com/Azure/azure-container-networking/store" -) - -const ( - // Service name. - name = "azure-cnimonitor" - pluginName = "azure-vnet" - DEFAULT_TIMEOUT_IN_SECS = "10" -) - -// Version is populated by make during build. -var version string - -// Command line arguments for CNM plugin. -var args = acn.ArgumentList{ - { - Name: acn.OptLogLevel, - Shorthand: acn.OptLogLevelAlias, - Description: "Set the logging level", - Type: "int", - DefaultValue: acn.OptLogLevelInfo, - ValueMap: map[string]interface{}{ - acn.OptLogLevelInfo: log.LevelInfo, - acn.OptLogLevelDebug: log.LevelDebug, - }, - }, - { - Name: acn.OptLogTarget, - Shorthand: acn.OptLogTargetAlias, - Description: "Set the logging target", - Type: "int", - DefaultValue: acn.OptLogTargetFile, - ValueMap: map[string]interface{}{ - acn.OptLogTargetSyslog: log.TargetSyslog, - acn.OptLogTargetStderr: log.TargetStderr, - acn.OptLogTargetFile: log.TargetLogfile, - }, - }, - { - Name: acn.OptLogLocation, - Shorthand: acn.OptLogLocationAlias, - Description: "Set the directory location where logs will be saved", - Type: "string", - DefaultValue: "", - }, - { - Name: acn.OptIntervalTime, - Shorthand: acn.OptIntervalTimeAlias, - Description: "Periodic Interval Time", - Type: "int", - DefaultValue: DEFAULT_TIMEOUT_IN_SECS, - }, - { - Name: acn.OptVersion, - Shorthand: acn.OptVersionAlias, - Description: "Print version information", - Type: "bool", - DefaultValue: false, - }, -} - -// Prints description and version information. -func printVersion() { - fmt.Printf("Azure Container Network Service\n") - fmt.Printf("Version %v\n", version) -} - -// Main is the entry point for CNS. -func main() { - // Initialize and parse command line arguments. - acn.ParseArgs(&args, printVersion) - - logLevel := acn.GetArg(acn.OptLogLevel).(int) - logTarget := acn.GetArg(acn.OptLogTarget).(int) - logDirectory := acn.GetArg(acn.OptLogLocation).(string) - timeout := acn.GetArg(acn.OptIntervalTime).(int) - vers := acn.GetArg(acn.OptVersion).(bool) - - if vers { - printVersion() - os.Exit(0) - } - - // Initialize CNS. - var config acn.PluginConfig - config.Version = version - - // Create a channel to receive unhandled errors from CNS. - config.ErrChan = make(chan error, 1) - - var err error - // Create logging provider. - log.SetName(name) - log.SetLevel(logLevel) - if err := log.SetTargetLogDirectory(logTarget, logDirectory); err != nil { - fmt.Printf("[monitor] Failed to configure logging: %v\n", err) - return - } - - // Log platform information. - log.Printf("[monitor] Running on %v", platform.GetOSInfo()) - - netMonitor := &cnms.NetworkMonitor{ - AddRulesToBeValidated: make(map[string]int), - DeleteRulesToBeValidated: make(map[string]int), - } - - for true { - config.Store, err = store.NewJsonFileStore(platform.CNIRuntimePath + pluginName + ".json") - if err != nil { - fmt.Printf("[monitor] Failed to create store: %v\n", err) - return - } - - nm, err := network.NewNetworkManager() - if err != nil { - log.Printf("[monitor] Failed while creating network manager") - return - } - - if err := nm.Initialize(&config); err != nil { - log.Printf("[monitor] Failed while initializing network manager %+v", err) - } - - log.Printf("[monitor] network manager:%+v", nm) - - if err := nm.SetupNetworkUsingState(netMonitor); err != nil { - log.Printf("[monitor] Failed while calling SetupNetworkUsingState with error %v", err) - return - } - - log.Printf("[monitor] Going to sleep for %v seconds", timeout) - time.Sleep(time.Duration(timeout) * time.Second) - nm = nil - } - - log.Close() -} From 99a5c5af8cce59773273335b6d3eda2642f7d038 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Fri, 13 Mar 2020 12:54:39 -0700 Subject: [PATCH 5/6] Made spacing changes suggested by Tamilmani --- cni/network/network_linux.go | 1 - cnms/cnmspackage/monitor2rules_linux.go | 3 +++ cnms/service/networkmonitor_test.go | 1 - ebtables/ebtables.go | 7 +++---- network/monitor_linux.go | 1 + 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 81c8e31eb4..263a30ae31 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -18,7 +18,6 @@ import ( const ( snatInterface = "eth1" infraInterface = "eth2" - binPath = "/opt/cni/bin" ) const ( diff --git a/cnms/cnmspackage/monitor2rules_linux.go b/cnms/cnmspackage/monitor2rules_linux.go index 9d2440bdfa..f8275c4ae8 100644 --- a/cnms/cnmspackage/monitor2rules_linux.go +++ b/cnms/cnmspackage/monitor2rules_linux.go @@ -63,6 +63,7 @@ func (networkMonitor *NetworkMonitor) CreateRequiredL2Rules( delete(networkMonitor.AddRulesToBeValidated, rule) } } + networkMonitor.addRulesNotExistInMap(currentStateRulesMap, currentEbtableRulesMap) return nil @@ -78,6 +79,7 @@ func (networkMonitor *NetworkMonitor) RemoveInvalidL2Rules( delete(networkMonitor.DeleteRulesToBeValidated, rule) } } + networkMonitor.deleteRulesNotExistInMap(currentEbtableRulesMap, currentStateRulesMap) return nil @@ -106,6 +108,7 @@ func GetEbTableRulesInMap() (map[string]string, error) { if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PreRouting); err != nil { return nil, err } + if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PostRouting); err != nil { return nil, err } diff --git a/cnms/service/networkmonitor_test.go b/cnms/service/networkmonitor_test.go index 760ccfa23c..b3c6700d7f 100644 --- a/cnms/service/networkmonitor_test.go +++ b/cnms/service/networkmonitor_test.go @@ -25,7 +25,6 @@ func TestMain(m *testing.M) { func addStateRulesToMap() map[string]string { rulesMap := make(map[string]string) - for _, value := range stateMapkey { rulesMap[value] = ebtables.PreRouting } diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go index 3dceb5f195..119c7e494f 100644 --- a/ebtables/ebtables.go +++ b/ebtables/ebtables.go @@ -12,10 +12,10 @@ import ( ) const ( - // Ebtables actions. + // Ebtable actions. Append = "-A" Delete = "-D" - // Ebtables tables. + // Ebtable tables. Nat = "nat" Broute = "broute" // Ebtable chains. @@ -71,8 +71,7 @@ func SetVepaMode(bridgeName string, downstreamIfNamePrefix string, upstreamMacAd if !strings.HasPrefix(bridgeName, downstreamIfNamePrefix) { rule := fmt.Sprintf("-i %s -j dnat --to-dst %s --dnat-target ACCEPT", bridgeName, upstreamMacAddress) - err := runEbCmd(table, action, chain, rule) - if err != nil { + if err := runEbCmd(table, action, chain, rule); err != nil { return err } } diff --git a/network/monitor_linux.go b/network/monitor_linux.go index 4c85bb99c6..4de1e76540 100644 --- a/network/monitor_linux.go +++ b/network/monitor_linux.go @@ -15,6 +15,7 @@ func (nm *networkManager) monitorNetworkState(networkMonitor *cnms.NetworkMonito log.Printf("GetEbTableRulesInMap failed with error %v", err) return err } + currentStateRulesMap := nm.AddStateRulesToMap() networkMonitor.CreateRequiredL2Rules(currentEbtableRulesMap, currentStateRulesMap) networkMonitor.RemoveInvalidL2Rules(currentEbtableRulesMap, currentStateRulesMap) From 19e67dd02936d809931ca11a0c57af34bf301e89 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Wed, 18 Mar 2020 12:18:49 -0700 Subject: [PATCH 6/6] Triggering new tests to run --- cnms/cnmspackage/monitor2rules_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cnms/cnmspackage/monitor2rules_linux.go b/cnms/cnmspackage/monitor2rules_linux.go index f8275c4ae8..f2180b83d3 100644 --- a/cnms/cnmspackage/monitor2rules_linux.go +++ b/cnms/cnmspackage/monitor2rules_linux.go @@ -21,7 +21,7 @@ func (networkMonitor *NetworkMonitor) deleteRulesNotExistInMap(chainRules map[st delete(networkMonitor.DeleteRulesToBeValidated, rule) } else { - log.Printf("[DELETE] Found unmatched rule chain %v rule %v itr %d. Giving one more iteration", chain, rule, itr) + log.Printf("[DELETE] Found unmatched rule chain %v rule %v itr %d. Giving one more iteration.", chain, rule, itr) networkMonitor.DeleteRulesToBeValidated[rule] = itr + 1 } }