diff --git a/Makefile b/Makefile index 183977dcdc..3b87f6d678 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,12 @@ CNSFILES = \ $(COREFILES) \ $(CNMFILES) +CNMSFILES = \ + $(wildcard cnms/*.go) \ + $(wildcard cnms/service/*.go) \ + $(wildcard cnms/cnmspackage/*.go) \ + $(COREFILES) + NPMFILES = \ $(wildcard npm/*.go) \ $(wildcard npm/ipsm/*.go) \ @@ -71,6 +77,7 @@ CNI_IPAM_DIR = cni/ipam/plugin CNI_TELEMETRY_DIR = cni/telemetry/service TELEMETRY_CONF_DIR = telemetry CNS_DIR = cns/service +CNMS_DIR = cnms/service NPM_DIR = npm/plugin OUTPUT_DIR = output BUILD_DIR = $(OUTPUT_DIR)/$(GOOS)_$(GOARCH) @@ -78,6 +85,7 @@ CNM_BUILD_DIR = $(BUILD_DIR)/cnm CNI_BUILD_DIR = $(BUILD_DIR)/cni CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy CNS_BUILD_DIR = $(BUILD_DIR)/cns +CNMS_BUILD_DIR = $(BUILD_DIR)/cnms NPM_BUILD_DIR = $(BUILD_DIR)/npm NPM_TELEMETRY_DIR = $(NPM_BUILD_DIR)/telemetry CNI_AI_ID = 5515a1eb-b2bc-406a-98eb-ba462e6f0411 @@ -107,6 +115,7 @@ CNM_ARCHIVE_NAME = azure-vnet-cnm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNI_ARCHIVE_NAME = azure-vnet-cni-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNI_MULTITENANCY_ARCHIVE_NAME = azure-vnet-cni-multitenancy-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) CNS_ARCHIVE_NAME = azure-cns-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) +CNMS_ARCHIVE_NAME = azure-cnms-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) NPM_ARCHIVE_NAME = azure-npm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) NPM_IMAGE_ARCHIVE_NAME = azure-npm-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) TELEMETRY_IMAGE_ARCHIVE_NAME = azure-vnet-telemetry-$(GOOS)-$(GOARCH)-$(VERSION).$(ARCHIVE_EXT) @@ -140,11 +149,12 @@ azure-vnet-telemetry: $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) # Azure-NPM only supports Linux for now. ifeq ($(GOOS),linux) +azure-cnms: $(CNMS_BUILD_DIR)/azure-cnms$(EXE_EXT) cnms-archive azure-npm: $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT) npm-archive endif ifeq ($(GOOS),linux) -all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-npm +all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-cnms azure-npm else all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns endif @@ -181,6 +191,10 @@ $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT): $(CNIFILES) $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT): $(CNSFILES) go build -v -o $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -X $(cnsaipath)=$(CNS_AI_ID) -s -w" $(CNS_DIR)/*.go +# Build the Azure CNMS Service. +$(CNMS_BUILD_DIR)/azure-cnms$(EXE_EXT): $(CNMSFILES) + go build -v -o $(CNMS_BUILD_DIR)/azure-cnms$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNMS_DIR)/*.go + # Build the Azure NPM plugin. $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT): $(NPMFILES) go build -v -o $(NPM_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNI_TELEMETRY_DIR)/*.go @@ -325,6 +339,15 @@ cns-archive: cd $(CNS_BUILD_DIR) && $(ARCHIVE_CMD) $(CNS_ARCHIVE_NAME) azure-cns$(EXE_EXT) cns_config.json chown $(BUILD_USER):$(BUILD_USER) $(CNS_BUILD_DIR)/$(CNS_ARCHIVE_NAME) +# Create a CNMS archive for the target platform. Only Linux is supported for now. +.PHONY: cnms-archive +cnms-archive: +ifeq ($(GOOS),linux) + chmod 0755 $(CNMS_BUILD_DIR)/azure-cnms$(EXE_EXT) + cd $(CNMS_BUILD_DIR) && $(ARCHIVE_CMD) $(CNMS_ARCHIVE_NAME) azure-cnms$(EXE_EXT) + chown $(BUILD_USER):$(BUILD_USER) $(CNMS_BUILD_DIR)/$(CNMS_ARCHIVE_NAME) +endif + # Create a NPM archive for the target platform. Only Linux is supported for now. .PHONY: npm-archive npm-archive: diff --git a/cnms/Dockerfile b/cnms/Dockerfile new file mode 100644 index 0000000000..1c3f0da745 --- /dev/null +++ b/cnms/Dockerfile @@ -0,0 +1,7 @@ +FROM ubuntu:latest +RUN apt -y update +RUN apt-get -y upgrade +RUN apt install -y ebtables +RUN apt install -y net-tools +COPY networkmonitor /usr/bin/networkmonitor +CMD ["/usr/bin/networkmonitor"] \ No newline at end of file diff --git a/cnms/cnmspackage/api.go b/cnms/cnmspackage/api.go new file mode 100644 index 0000000000..01979756e4 --- /dev/null +++ b/cnms/cnmspackage/api.go @@ -0,0 +1,6 @@ +package cnms + +type NetworkMonitor struct { + AddRulesToBeValidated map[string]int + DeleteRulesToBeValidated map[string]int +} diff --git a/cnms/cnmspackage/monitor2rules_linux.go b/cnms/cnmspackage/monitor2rules_linux.go new file mode 100644 index 0000000000..f2180b83d3 --- /dev/null +++ b/cnms/cnmspackage/monitor2rules_linux.go @@ -0,0 +1,117 @@ +package cnms + +import ( + "github.com/Azure/azure-container-networking/ebtables" + "github.com/Azure/azure-container-networking/log" +) + +// deleteRulesNotExistInMap deletes rules from nat Ebtable if rule was not in stateRules after a certain number of iterations. +func (networkMonitor *NetworkMonitor) deleteRulesNotExistInMap(chainRules map[string]string, stateRules map[string]string) { + + table := ebtables.Nat + action := ebtables.Delete + + for rule, chain := range chainRules { + if _, ok := stateRules[rule]; !ok { + if itr, ok := networkMonitor.DeleteRulesToBeValidated[rule]; ok && itr > 0 { + log.Printf("[monitor] Deleting Ebtable rule as it didn't exist in state for %d iterations chain %v rule %v", itr, chain, rule) + if err := ebtables.SetEbRule(table, action, chain, rule); err != nil { + log.Printf("[monitor] Error while deleting ebtable rule %v", err) + } + + delete(networkMonitor.DeleteRulesToBeValidated, rule) + } else { + log.Printf("[DELETE] Found unmatched rule chain %v rule %v itr %d. Giving one more iteration.", chain, rule, itr) + networkMonitor.DeleteRulesToBeValidated[rule] = itr + 1 + } + } + } +} + +// addRulesNotExistInMap adds rules to nat Ebtable if rule was in stateRules and not in current chain rules after a certain number of iterations. +func (networkMonitor *NetworkMonitor) addRulesNotExistInMap( + stateRules map[string]string, + chainRules map[string]string) { + + table := ebtables.Nat + action := ebtables.Append + + for rule, chain := range stateRules { + if _, ok := chainRules[rule]; !ok { + if itr, ok := networkMonitor.AddRulesToBeValidated[rule]; ok && itr > 0 { + log.Printf("[monitor] Adding Ebtable rule as it existed in state rules but not in current chain rules for %d iterations chain %v rule %v", itr, chain, rule) + if err := ebtables.SetEbRule(table, action, chain, rule); err != nil { + log.Printf("[monitor] Error while adding ebtable rule %v", err) + } + + delete(networkMonitor.AddRulesToBeValidated, rule) + } else { + log.Printf("[ADD] Found unmatched rule chain %v rule %v itr %d. Giving one more iteration", chain, rule, itr) + networkMonitor.AddRulesToBeValidated[rule] = itr + 1 + } + } + } +} + +// CreateRequiredL2Rules finds the rules that should be in nat ebtable based on state. +func (networkMonitor *NetworkMonitor) CreateRequiredL2Rules( + currentEbtableRulesMap map[string]string, + currentStateRulesMap map[string]string) error { + + for rule := range networkMonitor.AddRulesToBeValidated { + if _, ok := currentStateRulesMap[rule]; !ok { + delete(networkMonitor.AddRulesToBeValidated, rule) + } + } + + networkMonitor.addRulesNotExistInMap(currentStateRulesMap, currentEbtableRulesMap) + + return nil +} + +// RemoveInvalidL2Rules removes rules that should not be in nat ebtable based on state. +func (networkMonitor *NetworkMonitor) RemoveInvalidL2Rules( + currentEbtableRulesMap map[string]string, + currentStateRulesMap map[string]string) error { + + for rule := range networkMonitor.DeleteRulesToBeValidated { + if _, ok := currentEbtableRulesMap[rule]; !ok { + delete(networkMonitor.DeleteRulesToBeValidated, rule) + } + } + + networkMonitor.deleteRulesNotExistInMap(currentEbtableRulesMap, currentStateRulesMap) + + return nil +} + +// generateL2RulesMap gets rules from chainName and puts them in currentEbtableRulesMap. +func generateL2RulesMap(currentEbtableRulesMap map[string]string, chainName string) error { + table := ebtables.Nat + rules, err := ebtables.GetEbtableRules(table, chainName) + if err != nil { + log.Printf("[monitor] Error while getting rules list from table %v chain %v. Error: %v", + table, chainName, err) + return err + } + + for _, rule := range rules { + currentEbtableRulesMap[rule] = chainName + } + + return nil +} + +// GetEbTableRulesInMap gathers prerouting and postrouting rules into a map. +func GetEbTableRulesInMap() (map[string]string, error) { + currentEbtableRulesMap := make(map[string]string) + if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PreRouting); err != nil { + return nil, err + } + + if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PostRouting); err != nil { + return nil, err + } + + return currentEbtableRulesMap, nil +} diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go new file mode 100644 index 0000000000..a75bae2b94 --- /dev/null +++ b/cnms/service/networkmonitor.go @@ -0,0 +1,150 @@ +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package main + +import ( + "fmt" + "os" + "time" + + cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" + acn "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network" + "github.com/Azure/azure-container-networking/platform" + "github.com/Azure/azure-container-networking/store" +) + +const ( + // Service name. + name = "azure-cnimonitor" + pluginName = "azure-vnet" + DEFAULT_TIMEOUT_IN_SECS = "10" +) + +// Version is populated by make during build. +var version string + +// Command line arguments for CNM plugin. +var args = acn.ArgumentList{ + { + Name: acn.OptLogLevel, + Shorthand: acn.OptLogLevelAlias, + Description: "Set the logging level", + Type: "int", + DefaultValue: acn.OptLogLevelInfo, + ValueMap: map[string]interface{}{ + acn.OptLogLevelInfo: log.LevelInfo, + acn.OptLogLevelDebug: log.LevelDebug, + }, + }, + { + Name: acn.OptLogTarget, + Shorthand: acn.OptLogTargetAlias, + Description: "Set the logging target", + Type: "int", + DefaultValue: acn.OptLogTargetFile, + ValueMap: map[string]interface{}{ + acn.OptLogTargetSyslog: log.TargetSyslog, + acn.OptLogTargetStderr: log.TargetStderr, + acn.OptLogTargetFile: log.TargetLogfile, + }, + }, + { + Name: acn.OptLogLocation, + Shorthand: acn.OptLogLocationAlias, + Description: "Set the directory location where logs will be saved", + Type: "string", + DefaultValue: "", + }, + { + Name: acn.OptIntervalTime, + Shorthand: acn.OptIntervalTimeAlias, + Description: "Periodic Interval Time", + Type: "int", + DefaultValue: DEFAULT_TIMEOUT_IN_SECS, + }, + { + Name: acn.OptVersion, + Shorthand: acn.OptVersionAlias, + Description: "Print version information", + Type: "bool", + DefaultValue: false, + }, +} + +// Prints description and version information. +func printVersion() { + fmt.Printf("Azure Container Network Monitoring Service\n") + fmt.Printf("Version %v\n", version) +} + +// Main is the entry point for CNMS. +func main() { + // Initialize and parse command line arguments. + acn.ParseArgs(&args, printVersion) + logLevel := acn.GetArg(acn.OptLogLevel).(int) + logTarget := acn.GetArg(acn.OptLogTarget).(int) + logDirectory := acn.GetArg(acn.OptLogLocation).(string) + timeout := acn.GetArg(acn.OptIntervalTime).(int) + vers := acn.GetArg(acn.OptVersion).(bool) + if vers { + printVersion() + os.Exit(0) + } + + // Initialize CNMS. + var config acn.PluginConfig + config.Version = version + + // Create a channel to receive unhandled errors from CNMS. + config.ErrChan = make(chan error, 1) + + var err error + // Create logging provider. + log.SetName(name) + log.SetLevel(logLevel) + if err := log.SetTargetLogDirectory(logTarget, logDirectory); err != nil { + fmt.Printf("[monitor] Failed to configure logging: %v\n", err) + return + } + + // Log platform information. + log.Printf("[monitor] Running on %v", platform.GetOSInfo()) + + netMonitor := &cnms.NetworkMonitor{ + AddRulesToBeValidated: make(map[string]int), + DeleteRulesToBeValidated: make(map[string]int), + } + + for true { + config.Store, err = store.NewJsonFileStore(platform.CNIRuntimePath + pluginName + ".json") + if err != nil { + fmt.Printf("[monitor] Failed to create store: %v\n", err) + return + } + + nm, err := network.NewNetworkManager() + if err != nil { + log.Printf("[monitor] Failed while creating network manager") + return + } + + if err := nm.Initialize(&config); err != nil { + log.Printf("[monitor] Failed while initializing network manager %+v", err) + } + + log.Printf("[monitor] network manager:%+v", nm) + + if err := nm.SetupNetworkUsingState(netMonitor); err != nil { + log.Printf("[monitor] Failed while calling SetupNetworkUsingState with error %v", err) + } + + log.Printf("[monitor] Going to sleep for %v seconds", timeout) + time.Sleep(time.Duration(timeout) * time.Second) + nm = nil + } + + log.Close() +} diff --git a/cnms/service/networkmonitor_test.go b/cnms/service/networkmonitor_test.go new file mode 100644 index 0000000000..b3c6700d7f --- /dev/null +++ b/cnms/service/networkmonitor_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "os" + "testing" + + cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" + "github.com/Azure/azure-container-networking/ebtables" +) + +const ( + test1 = "test1.json" + test2 = "test2.json" +) + +var stateMapkey []string + +func TestMain(m *testing.M) { + stateMapkey = append(stateMapkey, "-p ARP -i eth0 --arp-op Reply -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT") + stateMapkey = append(stateMapkey, "-p ARP --arp-op Request --arp-ip-dst 10.240.0.6 -j arpreply --arpreply-mac cc:ad:1d:4e:e5:f1") + stateMapkey = append(stateMapkey, "-p IPv4 -i eth0 --ip-dst 10.240.0.6 -j dnat --to-dst cc:ad:1d:4e:e5:f1 --dnat-target ACCEPT") + exitCode := m.Run() + os.Exit(exitCode) +} + +func addStateRulesToMap() map[string]string { + rulesMap := make(map[string]string) + for _, value := range stateMapkey { + rulesMap[value] = ebtables.PreRouting + } + + rulesMap["-s Unicast -o eth0 -j snat --to-src 00:0d:12:3a:5d:32 --snat-arp --snat-target ACCEPT"] = ebtables.PostRouting + + return rulesMap +} + +func TestAddMissingRule(t *testing.T) { + netMonitor := &cnms.NetworkMonitor{ + AddRulesToBeValidated: make(map[string]int), + DeleteRulesToBeValidated: make(map[string]int), + } + + currentStateRulesMap := addStateRulesToMap() + currentEbTableRulesMap := make(map[string]string) + testKey := "" + + for key, value := range currentStateRulesMap { + if value != ebtables.PostRouting { + currentEbTableRulesMap[key] = value + } else { + testKey = key + } + } + + netMonitor.CreateRequiredL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.AddRulesToBeValidated) != 1 { + t.Fatalf("Expected AddRulesToBeValidated length to be 1 but got %v", len(netMonitor.AddRulesToBeValidated)) + } + + netMonitor.RemoveInvalidL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.DeleteRulesToBeValidated) != 0 { + t.Fatalf("Expected DeleteRulesToBeValidated length to be 0 but got %v", len(netMonitor.DeleteRulesToBeValidated)) + } + + for key, value := range netMonitor.AddRulesToBeValidated { + if key != testKey { + t.Fatalf("Expected AzurePostRouting snat but got %v", value) + } + } + + netMonitor.CreateRequiredL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.AddRulesToBeValidated) != 0 { + t.Fatalf("Expected AddRulesToBeValidated length to be 0 but got %v", len(netMonitor.AddRulesToBeValidated)) + } +} + +func TestDeleteInvalidRule(t *testing.T) { + netMonitor := &cnms.NetworkMonitor{ + AddRulesToBeValidated: make(map[string]int), + DeleteRulesToBeValidated: make(map[string]int), + } + + currentStateRulesMap := addStateRulesToMap() + currentEbTableRulesMap := make(map[string]string) + + for key, value := range currentStateRulesMap { + currentEbTableRulesMap[key] = value + } + + delete(currentStateRulesMap, stateMapkey[0]) + + netMonitor.CreateRequiredL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.AddRulesToBeValidated) != 0 { + t.Fatalf("Expected AddRulesToBeValidated length to be 0 but got %v", len(netMonitor.AddRulesToBeValidated)) + } + + netMonitor.RemoveInvalidL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.DeleteRulesToBeValidated) != 1 { + t.Fatalf("Expected DeleteRulesToBeValidated length to be 1 but got %v", len(netMonitor.DeleteRulesToBeValidated)) + } + + for key, value := range netMonitor.AddRulesToBeValidated { + if key != stateMapkey[0] { + t.Fatalf("Expected %v but got %v", stateMapkey[0], value) + } + } + + netMonitor.RemoveInvalidL2Rules(currentEbTableRulesMap, currentStateRulesMap) + if len(netMonitor.DeleteRulesToBeValidated) != 0 { + t.Fatalf("Expected DeleteRulesToBeValidated length to be 0 but got %v", len(netMonitor.DeleteRulesToBeValidated)) + } +} diff --git a/common/config.go b/common/config.go index a7f5d42330..71f0495926 100644 --- a/common/config.go +++ b/common/config.go @@ -53,6 +53,11 @@ const ( OptReportToHostInterval = "report-interval" OptReportToHostIntervalAlias = "hostinterval" + // Periodic Interval Time + OptIntervalTime = "interval" + OptIntervalTimeAlias = "it" + OptDefaultIntervalTime = "defaultinterval" + // Version. OptVersion = "version" OptVersionAlias = "v" diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go index ef516d42a7..119c7e494f 100644 --- a/ebtables/ebtables.go +++ b/ebtables/ebtables.go @@ -5,61 +5,96 @@ package ebtables import ( "fmt" - "io/ioutil" "net" - "os/exec" "strings" - "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" ) const ( - // Ebtables actions. + // Ebtable actions. Append = "-A" Delete = "-D" + // Ebtable tables. + Nat = "nat" + Broute = "broute" + // Ebtable chains. + PreRouting = "PREROUTING" + PostRouting = "POSTROUTING" + Brouting = "BROUTING" ) -// InstallEbtables installs the ebtables package. -func installEbtables() { - version, _ := ioutil.ReadFile("/proc/version") - os := strings.ToLower(string(version)) - - if strings.Contains(os, "ubuntu") { - executeShellCommand("apt-get install ebtables") - } else if strings.Contains(os, "redhat") { - executeShellCommand("yum install ebtables") - } else { - log.Printf("Unable to detect OS platform. Please make sure the ebtables package is installed.") - } -} - // SetSnatForInterface sets a MAC SNAT rule for an interface. func SetSnatForInterface(interfaceName string, macAddress net.HardwareAddr, action string) error { - command := fmt.Sprintf( - "ebtables -t nat %s POSTROUTING -s unicast -o %s -j snat --to-src %s --snat-arp --snat-target ACCEPT", - action, interfaceName, macAddress.String()) + table := Nat + chain := PostRouting + rule := fmt.Sprintf("-s unicast -o %s -j snat --to-src %s --snat-arp --snat-target ACCEPT", + interfaceName, macAddress.String()) - return executeShellCommand(command) + return runEbCmd(table, action, chain, rule) } // SetArpReply sets an ARP reply rule for the given target IP address and MAC address. func SetArpReply(ipAddress net.IP, macAddress net.HardwareAddr, action string) error { - command := fmt.Sprintf( - "ebtables -t nat %s PREROUTING -p ARP --arp-op Request --arp-ip-dst %s -j arpreply --arpreply-mac %s --arpreply-target DROP", - action, ipAddress, macAddress.String()) + table := Nat + chain := PreRouting + rule := fmt.Sprintf("-p ARP --arp-op Request --arp-ip-dst %s -j arpreply --arpreply-mac %s --arpreply-target DROP", + ipAddress, macAddress.String()) - return executeShellCommand(command) + return runEbCmd(table, action, chain, rule) } // SetBrouteAccept sets an EB rule. func SetBrouteAccept(ipAddress, action string) error { - command := fmt.Sprintf( - "ebtables -t broute %s BROUTING --ip-dst %s -p IPv4 -j redirect --redirect-target ACCEPT", - action, ipAddress) - _, err := platform.ExecuteCommand(command) + table := Broute + chain := Brouting + rule := fmt.Sprintf("--ip-dst %s -p IPv4 -j redirect --redirect-target ACCEPT", ipAddress) - return err + return runEbCmd(table, action, chain, rule) +} + +// SetDnatForArpReplies sets a MAC DNAT rule for ARP replies received on an interface. +func SetDnatForArpReplies(interfaceName string, action string) error { + table := Nat + chain := PreRouting + rule := fmt.Sprintf("-p ARP -i %s --arp-op Reply -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT", + interfaceName) + + return runEbCmd(table, action, chain, rule) +} + +// SetVepaMode sets the VEPA mode for a bridge and its ports. +func SetVepaMode(bridgeName string, downstreamIfNamePrefix string, upstreamMacAddress string, action string) error { + table := Nat + chain := PreRouting + + if !strings.HasPrefix(bridgeName, downstreamIfNamePrefix) { + rule := fmt.Sprintf("-i %s -j dnat --to-dst %s --dnat-target ACCEPT", bridgeName, upstreamMacAddress) + + if err := runEbCmd(table, action, chain, rule); err != nil { + return err + } + } + + rule2 := fmt.Sprintf("-i %s+ -j dnat --to-dst %s --dnat-target ACCEPT", + downstreamIfNamePrefix, upstreamMacAddress) + + return runEbCmd(table, action, chain, rule2) +} + +// SetDnatForIPAddress sets a MAC DNAT rule for an IP address. +func SetDnatForIPAddress(interfaceName string, ipAddress net.IP, macAddress net.HardwareAddr, action string) error { + table := Nat + chain := PreRouting + rule := fmt.Sprintf("-p IPv4 -i %s --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT", + interfaceName, ipAddress.String(), macAddress.String()) + + return runEbCmd(table, action, chain, rule) +} + +// SetEbRule sets any given eb rule +func SetEbRule(table, action, chain, rule string) error { + return runEbCmd(table, action, chain, rule) } // GetEbtableRules gets EB rules for a table and chain. @@ -114,50 +149,10 @@ func EbTableRuleExists(tableName, chainName, matchSet string) (bool, error) { return false, nil } -// SetDnatForArpReplies sets a MAC DNAT rule for ARP replies received on an interface. -func SetDnatForArpReplies(interfaceName string, action string) error { - command := fmt.Sprintf( - "ebtables -t nat %s PREROUTING -p ARP -i %s --arp-op Reply -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT", - action, interfaceName) - - return executeShellCommand(command) -} - -// SetVepaMode sets the VEPA mode for a bridge and its ports. -func SetVepaMode(bridgeName string, downstreamIfNamePrefix string, upstreamMacAddress string, action string) error { - if !strings.HasPrefix(bridgeName, downstreamIfNamePrefix) { - command := fmt.Sprintf( - "ebtables -t nat %s PREROUTING -i %s -j dnat --to-dst %s --dnat-target ACCEPT", - action, bridgeName, upstreamMacAddress) - - err := executeShellCommand(command) - if err != nil { - return err - } - } - - command := fmt.Sprintf( - "ebtables -t nat %s PREROUTING -i %s+ -j dnat --to-dst %s --dnat-target ACCEPT", - action, downstreamIfNamePrefix, upstreamMacAddress) - - return executeShellCommand(command) -} - -// SetDnatForIPAddress sets a MAC DNAT rule for an IP address. -func SetDnatForIPAddress(interfaceName string, ipAddress net.IP, macAddress net.HardwareAddr, action string) error { - command := fmt.Sprintf( - "ebtables -t nat %s PREROUTING -p IPv4 -i %s --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT", - action, interfaceName, ipAddress.String(), macAddress.String()) - - return executeShellCommand(command) -} +// runEbCmd runs an EB rule command. +func runEbCmd(table, action, chain, rule string) error { + command := fmt.Sprintf("ebtables -t %s %s %s %s", table, action, chain, rule) + _, err := platform.ExecuteCommand(command) -func executeShellCommand(command string) error { - log.Debugf("[ebtables] %s", command) - cmd := exec.Command("sh", "-c", command) - err := cmd.Start() - if err != nil { - return err - } - return cmd.Wait() + return err } diff --git a/network/manager.go b/network/manager.go index 98bcbf5ace..18252be530 100644 --- a/network/manager.go +++ b/network/manager.go @@ -7,6 +7,7 @@ import ( "sync" "time" + cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" @@ -67,6 +68,7 @@ type NetworkManager interface { DetachEndpoint(networkId string, endpointId string) error UpdateEndpoint(networkId string, existingEpInfo *EndpointInfo, targetEpInfo *EndpointInfo) error GetNumberOfEndpoints(ifName string, networkId string) int + SetupNetworkUsingState(networkMonitor *cnms.NetworkMonitor) error } // Creates a new network manager. @@ -105,6 +107,9 @@ func (nm *networkManager) restore() error { // Ignore the persisted state if it is older than the last reboot time. // Read any persisted state. + nm.Lock() + defer nm.Unlock() + err := nm.store.Read(storeKey, nm) if err != nil { if err == store.ErrKeyNotFound { @@ -487,3 +492,7 @@ func (nm *networkManager) GetNumberOfEndpoints(ifName string, networkId string) return 0 } + +func (nm *networkManager) SetupNetworkUsingState(networkMonitor *cnms.NetworkMonitor) error { + return nm.monitorNetworkState(networkMonitor) +} diff --git a/network/monitor_linux.go b/network/monitor_linux.go new file mode 100644 index 0000000000..4de1e76540 --- /dev/null +++ b/network/monitor_linux.go @@ -0,0 +1,51 @@ +package network + +import ( + "fmt" + + cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" + "github.com/Azure/azure-container-networking/ebtables" + "github.com/Azure/azure-container-networking/log" +) + +// monitorNetworkState compares current ebtable nat rules with state rules and matches state. +func (nm *networkManager) monitorNetworkState(networkMonitor *cnms.NetworkMonitor) error { + currentEbtableRulesMap, err := cnms.GetEbTableRulesInMap() + if err != nil { + log.Printf("GetEbTableRulesInMap failed with error %v", err) + return err + } + + currentStateRulesMap := nm.AddStateRulesToMap() + networkMonitor.CreateRequiredL2Rules(currentEbtableRulesMap, currentStateRulesMap) + networkMonitor.RemoveInvalidL2Rules(currentEbtableRulesMap, currentStateRulesMap) + + return nil +} + +// AddStateRulesToMap adds rules to state based off network manager settings. +func (nm *networkManager) AddStateRulesToMap() map[string]string { + rulesMap := make(map[string]string) + + for _, extIf := range nm.ExternalInterfaces { + arpDnatKey := fmt.Sprintf("-p ARP -i %s --arp-op Reply -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT", extIf.Name) + rulesMap[arpDnatKey] = ebtables.PreRouting + + snatKey := fmt.Sprintf("-s Unicast -o %s -j snat --to-src %s --snat-arp --snat-target ACCEPT", extIf.Name, extIf.MacAddress.String()) + rulesMap[snatKey] = ebtables.PostRouting + + for _, nw := range extIf.Networks { + for _, ep := range nw.Endpoints { + for _, ipAddr := range ep.IPAddresses { + arpReplyKey := fmt.Sprintf("-p ARP --arp-op Request --arp-ip-dst %s -j arpreply --arpreply-mac %s", ipAddr.IP.String(), ep.MacAddress.String()) + rulesMap[arpReplyKey] = ebtables.PreRouting + + dnatMacKey := fmt.Sprintf("-p IPv4 -i %s --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT", extIf.Name, ipAddr.IP.String(), ep.MacAddress.String()) + rulesMap[dnatMacKey] = ebtables.PreRouting + } + } + } + } + + return rulesMap +} diff --git a/network/monitor_windows.go b/network/monitor_windows.go new file mode 100644 index 0000000000..a3b13d62d8 --- /dev/null +++ b/network/monitor_windows.go @@ -0,0 +1,3 @@ +func (nm *networkManager) monitorNetworkState() error { + return nil +} \ No newline at end of file