diff --git a/Makefile b/Makefile index 251110e3c3..38823b578d 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,6 @@ azure-cnm-plugin: cnm-binary cnm-archive azure-cni-plugin: azure-vnet-binary azure-vnet-ipam-binary azure-vnet-ipamv6-binary azure-vnet-telemetry-binary cni-archive azure-cns: azure-cns-binary cns-archive acncli: acncli-binary acncli-archive -azure-cnms: azure-cnms-binary cnms-archive azure-npm: azure-npm-binary npm-archive azure-ipam: azure-ipam-binary azure-ipam-archive diff --git a/cnms/Dockerfile b/cnms/Dockerfile deleted file mode 100644 index ff4dc05677..0000000000 --- a/cnms/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 -ARG CNMS_BUILD_DIR -RUN tdnf upgrade -RUN tdnf install -y ebtables -RUN tdnf install -y net-tools -COPY output/linux_amd64/cnms/azure-cnms /usr/bin/azure-cnms -RUN chmod +x /usr/bin/azure-cnms -CMD ["/usr/bin/azure-cnms"] diff --git a/cnms/cnmspackage/api.go b/cnms/cnmspackage/api.go deleted file mode 100644 index baf83e09cc..0000000000 --- a/cnms/cnmspackage/api.go +++ /dev/null @@ -1,9 +0,0 @@ -package cnms - -import "github.com/Azure/azure-container-networking/telemetry" - -type NetworkMonitor struct { - AddRulesToBeValidated map[string]int - DeleteRulesToBeValidated map[string]int - CNIReport *telemetry.CNIReport -} diff --git a/cnms/cnmspackage/monitor2rules_linux.go b/cnms/cnmspackage/monitor2rules_linux.go deleted file mode 100644 index 3b3ffb6a7f..0000000000 --- a/cnms/cnmspackage/monitor2rules_linux.go +++ /dev/null @@ -1,124 +0,0 @@ -package cnms - -import ( - "fmt" - - "github.com/Azure/azure-container-networking/ebtables" - "github.com/Azure/azure-container-networking/log" -) - -// deleteRulesNotExistInMap deletes rules from nat Ebtable if rule was not in stateRules after a certain number of iterations. -func (networkMonitor *NetworkMonitor) deleteRulesNotExistInMap(chainRules map[string]string, stateRules map[string]string) { - table := ebtables.Nat - action := ebtables.Delete - - for rule, chain := range chainRules { - if _, ok := stateRules[rule]; !ok { - if itr, ok := networkMonitor.DeleteRulesToBeValidated[rule]; ok && itr > 0 { - buf := fmt.Sprintf("[monitor] Deleting Ebtable rule as it didn't exist in state for %d iterations chain %v rule %v", itr, chain, rule) - if err := ebtables.SetEbRule(table, action, chain, rule); err != nil { - buf = fmt.Sprintf("[monitor] Error while deleting ebtable rule %v", err) - } - - log.Printf(buf) - networkMonitor.CNIReport.ErrorMessage = buf - networkMonitor.CNIReport.OperationType = "EBTableDelete" - delete(networkMonitor.DeleteRulesToBeValidated, rule) - } else { - log.Printf("[DELETE] Found unmatched rule chain %v rule %v itr %d. Giving one more iteration.", chain, rule, itr) - networkMonitor.DeleteRulesToBeValidated[rule] = itr + 1 - } - } - } -} - -// addRulesNotExistInMap adds rules to nat Ebtable if rule was in stateRules and not in current chain rules after a certain number of iterations. -func (networkMonitor *NetworkMonitor) addRulesNotExistInMap( - stateRules map[string]string, - chainRules map[string]string) { - - table := ebtables.Nat - action := ebtables.Append - - for rule, chain := range stateRules { - if _, ok := chainRules[rule]; !ok { - if itr, ok := networkMonitor.AddRulesToBeValidated[rule]; ok && itr > 0 { - buf := fmt.Sprintf("[monitor] Adding Ebtable rule as it existed in state rules but not in current chain rules for %d iterations chain %v rule %v", itr, chain, rule) - if err := ebtables.SetEbRule(table, action, chain, rule); err != nil { - buf = fmt.Sprintf("[monitor] Error while adding ebtable rule %v", err) - } - - log.Printf(buf) - networkMonitor.CNIReport.ErrorMessage = buf - networkMonitor.CNIReport.OperationType = "EBTableAdd" - delete(networkMonitor.AddRulesToBeValidated, rule) - } else { - log.Printf("[ADD] Found unmatched rule chain %v rule %v itr %d. Giving one more iteration.", chain, rule, itr) - networkMonitor.AddRulesToBeValidated[rule] = itr + 1 - } - } - } -} - -// CreateRequiredL2Rules finds the rules that should be in nat ebtable based on state. -func (networkMonitor *NetworkMonitor) CreateRequiredL2Rules( - currentEbtableRulesMap map[string]string, - currentStateRulesMap map[string]string) error { - - for rule := range networkMonitor.AddRulesToBeValidated { - if _, ok := currentStateRulesMap[rule]; !ok { - delete(networkMonitor.AddRulesToBeValidated, rule) - } - } - - networkMonitor.addRulesNotExistInMap(currentStateRulesMap, currentEbtableRulesMap) - - return nil -} - -// RemoveInvalidL2Rules removes rules that should not be in nat ebtable based on state. -func (networkMonitor *NetworkMonitor) RemoveInvalidL2Rules( - currentEbtableRulesMap map[string]string, - currentStateRulesMap map[string]string) error { - - for rule := range networkMonitor.DeleteRulesToBeValidated { - if _, ok := currentEbtableRulesMap[rule]; !ok { - delete(networkMonitor.DeleteRulesToBeValidated, rule) - } - } - - networkMonitor.deleteRulesNotExistInMap(currentEbtableRulesMap, currentStateRulesMap) - - return nil -} - -// generateL2RulesMap gets rules from chainName and puts them in currentEbtableRulesMap. -func generateL2RulesMap(currentEbtableRulesMap map[string]string, chainName string) error { - table := ebtables.Nat - rules, err := ebtables.GetEbtableRules(table, chainName) - if err != nil { - log.Printf("[monitor] Error while getting rules list from table %v chain %v. Error: %v", - table, chainName, err) - return err - } - - for _, rule := range rules { - currentEbtableRulesMap[rule] = chainName - } - - return nil -} - -// GetEbTableRulesInMap gathers prerouting and postrouting rules into a map. -func GetEbTableRulesInMap() (map[string]string, error) { - currentEbtableRulesMap := make(map[string]string) - if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PreRouting); err != nil { - return nil, err - } - - if err := generateL2RulesMap(currentEbtableRulesMap, ebtables.PostRouting); err != nil { - return nil, err - } - - return currentEbtableRulesMap, nil -} diff --git a/cnms/service/networkmonitor.go b/cnms/service/networkmonitor.go deleted file mode 100644 index 96c790a667..0000000000 --- a/cnms/service/networkmonitor.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2017 Microsoft. All rights reserved. -// MIT License - -package main - -import ( - "fmt" - "os" - "time" - - cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" - acn "github.com/Azure/azure-container-networking/common" - "github.com/Azure/azure-container-networking/iptables" - "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/netio" - "github.com/Azure/azure-container-networking/netlink" - "github.com/Azure/azure-container-networking/network" - "github.com/Azure/azure-container-networking/platform" - "github.com/Azure/azure-container-networking/processlock" - "github.com/Azure/azure-container-networking/store" - "github.com/Azure/azure-container-networking/telemetry" -) - -const ( - // Service name. - name = "azure-cnimonitor" - pluginName = "azure-vnet" - DEFAULT_TIMEOUT_IN_SECS = "10" - telemetryNumRetries = 5 - telemetryWaitTimeInMilliseconds = 200 -) - -// Version is populated by make during build. -var version string - -// Command line arguments for CNM plugin. -var args = acn.ArgumentList{ - { - Name: acn.OptLogLevel, - Shorthand: acn.OptLogLevelAlias, - Description: "Set the logging level", - Type: "int", - DefaultValue: acn.OptLogLevelInfo, - ValueMap: map[string]interface{}{ - acn.OptLogLevelInfo: log.LevelInfo, - acn.OptLogLevelDebug: log.LevelDebug, - }, - }, - { - Name: acn.OptLogTarget, - Shorthand: acn.OptLogTargetAlias, - Description: "Set the logging target", - Type: "int", - DefaultValue: acn.OptLogTargetFile, - ValueMap: map[string]interface{}{ - acn.OptLogTargetSyslog: log.TargetSyslog, - acn.OptLogTargetStderr: log.TargetStderr, - acn.OptLogTargetFile: log.TargetLogfile, - }, - }, - { - Name: acn.OptLogLocation, - Shorthand: acn.OptLogLocationAlias, - Description: "Set the directory location where logs will be saved", - Type: "string", - DefaultValue: "", - }, - { - Name: acn.OptIntervalTime, - Shorthand: acn.OptIntervalTimeAlias, - Description: "Periodic Interval Time", - Type: "int", - DefaultValue: DEFAULT_TIMEOUT_IN_SECS, - }, - { - Name: acn.OptVersion, - Shorthand: acn.OptVersionAlias, - Description: "Print version information", - Type: "bool", - DefaultValue: false, - }, -} - -// Prints description and version information. -func printVersion() { - fmt.Printf("Azure Container Network Monitoring Service\n") - fmt.Printf("Version %v\n", version) -} - -// Main is the entry point for CNMS. -func main() { - // Initialize and parse command line arguments. - acn.ParseArgs(&args, printVersion) - logLevel := acn.GetArg(acn.OptLogLevel).(int) - logTarget := acn.GetArg(acn.OptLogTarget).(int) - logDirectory := acn.GetArg(acn.OptLogLocation).(string) - timeout := acn.GetArg(acn.OptIntervalTime).(int) - vers := acn.GetArg(acn.OptVersion).(bool) - if vers { - printVersion() - os.Exit(0) - } - - // Initialize CNMS. - var config acn.PluginConfig - config.Version = version - - // Create a channel to receive unhandled errors from CNMS. - config.ErrChan = make(chan error, 1) - - var err error - // Create logging provider. - log.SetName(name) - log.SetLevel(logLevel) - if err := log.SetTargetLogDirectory(logTarget, logDirectory); err != nil { - fmt.Printf("[monitor] Failed to configure logging: %v\n", err) - return - } - - // Log platform information. - log.Printf("[monitor] Running on %v", platform.GetOSInfo()) - - reportManager := &telemetry.ReportManager{ - ContentType: telemetry.ContentType, - Report: &telemetry.CNIReport{ - Context: "AzureCNINetworkMonitor", - Version: version, - SystemDetails: telemetry.SystemInfo{}, - InterfaceDetails: telemetry.InterfaceInfo{}, - BridgeDetails: telemetry.BridgeInfo{}, - }, - } - - reportManager.Report.(*telemetry.CNIReport).GetOSDetails() - - netMonitor := &cnms.NetworkMonitor{ - AddRulesToBeValidated: make(map[string]int), - DeleteRulesToBeValidated: make(map[string]int), - CNIReport: reportManager.Report.(*telemetry.CNIReport), - } - - tb := telemetry.NewTelemetryBuffer(nil) - tb.ConnectToTelemetryService(telemetryNumRetries, telemetryWaitTimeInMilliseconds) - defer tb.Close() - - var lockclient processlock.Interface - for { - lockclient, err = processlock.NewFileLock(platform.CNILockPath + pluginName + store.LockExtension) - if err != nil { - log.Printf("Error initializing file lock:%v", err) - return - } - - config.Store, err = store.NewJsonFileStore(platform.CNIRuntimePath+pluginName+".json", lockclient, nil) - if err != nil { - fmt.Printf("[monitor] Failed to create store: %v\n", err) - return - } - - nl := netlink.NewNetlink() - nm, err := network.NewNetworkManager(nl, platform.NewExecClient(nil), &netio.NetIO{}, network.NewNamespaceClient(), iptables.NewClient()) - if err != nil { - log.Printf("[monitor] Failed while creating network manager") - return - } - - if err := nm.Initialize(&config, false); err != nil { - log.Printf("[monitor] Failed while initializing network manager %+v", err) - } - - log.Printf("[monitor] network manager:%+v", nm) - - if err := nm.SetupNetworkUsingState(netMonitor); err != nil { - log.Printf("[monitor] Failed while calling SetupNetworkUsingState with error %v", err) - } - - if netMonitor.CNIReport.ErrorMessage != "" { - log.Printf("[monitor] Reporting discrepancy in rules") - netMonitor.CNIReport.Timestamp = time.Now().Format("2006-01-02 15:04:05") - if err := reportManager.SendReport(tb); err != nil { - log.Errorf("[monitor] SendReport failed due to %v", err) - } else { - log.Printf("[monitor] Reported successfully") - } - - netMonitor.CNIReport.ErrorMessage = "" - } - - log.Printf("[monitor] Going to sleep for %v seconds", timeout) - time.Sleep(time.Duration(timeout) * time.Second) - nm = nil - } -} diff --git a/cnms/service/networkmonitor_linux_test.go b/cnms/service/networkmonitor_linux_test.go deleted file mode 100644 index 71887ee990..0000000000 --- a/cnms/service/networkmonitor_linux_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package main - -import ( - "os" - "testing" - - cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" - "github.com/Azure/azure-container-networking/ebtables" - "github.com/Azure/azure-container-networking/telemetry" -) - -const ( - test1 = "test1.json" - test2 = "test2.json" -) - -var stateMapkey []string - -func TestMain(m *testing.M) { - stateMapkey = append(stateMapkey, "-p ARP -i eth0 --arp-op Reply -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT") - stateMapkey = append(stateMapkey, "-p ARP --arp-op Request --arp-ip-dst 10.240.0.6 -j arpreply --arpreply-mac cc:ad:1d:4e:e5:f1") - stateMapkey = append(stateMapkey, "-p IPv4 -i eth0 --ip-dst 10.240.0.6 -j dnat --to-dst cc:ad:1d:4e:e5:f1 --dnat-target ACCEPT") - exitCode := m.Run() - os.Exit(exitCode) -} - -func addStateRulesToMap() map[string]string { - rulesMap := make(map[string]string) - for _, value := range stateMapkey { - rulesMap[value] = ebtables.PreRouting - } - - rulesMap["-s Unicast -o eth0 -j snat --to-src 00:0d:12:3a:5d:32 --snat-arp --snat-target ACCEPT"] = ebtables.PostRouting - - return rulesMap -} - -func TestAddMissingRule(t *testing.T) { - netMonitor := &cnms.NetworkMonitor{ - AddRulesToBeValidated: make(map[string]int), - DeleteRulesToBeValidated: make(map[string]int), - CNIReport: &telemetry.CNIReport{}, - } - - currentStateRulesMap := addStateRulesToMap() - currentEbTableRulesMap := make(map[string]string) - testKey := "" - - for key, value := range currentStateRulesMap { - if value != ebtables.PostRouting { - currentEbTableRulesMap[key] = value - } else { - testKey = key - } - } - - netMonitor.CreateRequiredL2Rules(currentEbTableRulesMap, currentStateRulesMap) - if len(netMonitor.AddRulesToBeValidated) != 1 { - t.Fatalf("Expected AddRulesToBeValidated length to be 1 but got %v", len(netMonitor.AddRulesToBeValidated)) - } - - netMonitor.RemoveInvalidL2Rules(currentEbTableRulesMap, currentStateRulesMap) - if len(netMonitor.DeleteRulesToBeValidated) != 0 { - t.Fatalf("Expected DeleteRulesToBeValidated length to be 0 but got %v", len(netMonitor.DeleteRulesToBeValidated)) - } - - for key, value := range netMonitor.AddRulesToBeValidated { - if key != testKey { - t.Fatalf("Expected AzurePostRouting snat but got %v", value) - } - } - - netMonitor.CreateRequiredL2Rules(currentEbTableRulesMap, currentStateRulesMap) - if len(netMonitor.AddRulesToBeValidated) != 0 { - t.Fatalf("Expected AddRulesToBeValidated length to be 0 but got %v", len(netMonitor.AddRulesToBeValidated)) - } -} - -func TestDeleteInvalidRule(t *testing.T) { - netMonitor := &cnms.NetworkMonitor{ - AddRulesToBeValidated: make(map[string]int), - DeleteRulesToBeValidated: make(map[string]int), - CNIReport: &telemetry.CNIReport{}, - } - - currentStateRulesMap := addStateRulesToMap() - currentEbTableRulesMap := make(map[string]string) - - for key, value := range currentStateRulesMap { - currentEbTableRulesMap[key] = value - } - - delete(currentStateRulesMap, stateMapkey[0]) - - netMonitor.CreateRequiredL2Rules(currentEbTableRulesMap, currentStateRulesMap) - if len(netMonitor.AddRulesToBeValidated) != 0 { - t.Fatalf("Expected AddRulesToBeValidated length to be 0 but got %v", len(netMonitor.AddRulesToBeValidated)) - } - - netMonitor.RemoveInvalidL2Rules(currentEbTableRulesMap, currentStateRulesMap) - if len(netMonitor.DeleteRulesToBeValidated) != 1 { - t.Fatalf("Expected DeleteRulesToBeValidated length to be 1 but got %v", len(netMonitor.DeleteRulesToBeValidated)) - } - - for key, value := range netMonitor.AddRulesToBeValidated { - if key != stateMapkey[0] { - t.Fatalf("Expected %v but got %v", stateMapkey[0], value) - } - } - - netMonitor.RemoveInvalidL2Rules(currentEbTableRulesMap, currentStateRulesMap) - if len(netMonitor.DeleteRulesToBeValidated) != 0 { - t.Fatalf("Expected DeleteRulesToBeValidated length to be 0 but got %v", len(netMonitor.DeleteRulesToBeValidated)) - } -} diff --git a/network/manager.go b/network/manager.go index 5064a7265a..a3af73d0c7 100644 --- a/network/manager.go +++ b/network/manager.go @@ -9,7 +9,6 @@ import ( "sync" "time" - cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" cnsclient "github.com/Azure/azure-container-networking/cns/client" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" @@ -108,7 +107,6 @@ type NetworkManager interface { DetachEndpoint(networkID string, endpointID string) error UpdateEndpoint(networkID string, existingEpInfo *EndpointInfo, targetEpInfo *EndpointInfo) error GetNumberOfEndpoints(ifName string, networkID string) int - SetupNetworkUsingState(networkMonitor *cnms.NetworkMonitor) error GetEndpointID(containerID, ifName string) string IsStatelessCNIMode() bool } @@ -675,10 +673,6 @@ func (nm *networkManager) GetNumberOfEndpoints(ifName string, networkId string) return 0 } -func (nm *networkManager) SetupNetworkUsingState(networkMonitor *cnms.NetworkMonitor) error { - return nm.monitorNetworkState(networkMonitor) -} - // GetEndpointID returns a unique endpoint ID based on the CNI mode. func (nm *networkManager) GetEndpointID(containerID, ifName string) string { if nm.IsStatelessCNIMode() { diff --git a/network/manager_mock.go b/network/manager_mock.go index 188d2bb2ad..316af8d5d3 100644 --- a/network/manager_mock.go +++ b/network/manager_mock.go @@ -1,7 +1,6 @@ package network import ( - cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" "github.com/Azure/azure-container-networking/common" ) @@ -131,11 +130,6 @@ func (nm *MockNetworkManager) GetNumberOfEndpoints(ifName string, networkID stri return 0 } -// SetupNetworkUsingState mock -func (nm *MockNetworkManager) SetupNetworkUsingState(networkMonitor *cnms.NetworkMonitor) error { - return nil -} - func (nm *MockNetworkManager) FindNetworkIDFromNetNs(netNs string) (string, error) { // based on the GetAllEndpoints func above, it seems that this mock is only intended to be used with // one network, so just return the network here if it exists diff --git a/network/monitor_linux.go b/network/monitor_linux.go deleted file mode 100644 index 3d437caafb..0000000000 --- a/network/monitor_linux.go +++ /dev/null @@ -1,74 +0,0 @@ -package network - -import ( - "fmt" - - cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" - "github.com/Azure/azure-container-networking/ebtables" - "go.uber.org/zap" -) - -const ( - ipv6Mask = "/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" -) - -// monitorNetworkState compares current ebtable nat rules with state rules and matches state. -func (nm *networkManager) monitorNetworkState(networkMonitor *cnms.NetworkMonitor) error { - currentEbtableRulesMap, err := cnms.GetEbTableRulesInMap() - if err != nil { - logger.Error("GetEbTableRulesInMap failed with", zap.Error(err)) - return err - } - - currentStateRulesMap := nm.AddStateRulesToMap() - networkMonitor.CreateRequiredL2Rules(currentEbtableRulesMap, currentStateRulesMap) - networkMonitor.RemoveInvalidL2Rules(currentEbtableRulesMap, currentStateRulesMap) - - return nil -} - -// AddStateRulesToMap adds rules to state based off network manager settings. -func (nm *networkManager) AddStateRulesToMap() map[string]string { - rulesMap := make(map[string]string) - - for _, extIf := range nm.ExternalInterfaces { - arpDnatKey := fmt.Sprintf("-p ARP -i %s --arp-op Reply -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT", extIf.Name) - rulesMap[arpDnatKey] = ebtables.PreRouting - - snatKey := fmt.Sprintf("-s Unicast -o %s -j snat --to-src %s --snat-arp --snat-target ACCEPT", extIf.Name, extIf.MacAddress.String()) - rulesMap[snatKey] = ebtables.PostRouting - - for _, extIP := range extIf.IPAddresses { - if extIP.IP.To4() != nil { - arpReplyKey := fmt.Sprintf("-p ARP --arp-op Request --arp-ip-dst %s -j arpreply --arpreply-mac %s", extIP.IP.String(), extIf.MacAddress.String()) - rulesMap[arpReplyKey] = ebtables.PreRouting - } - } - - for _, nw := range extIf.Networks { - for _, ep := range nw.Endpoints { - for _, ipAddr := range ep.IPAddresses { - if ipAddr.IP.To4() != nil { - arpReplyKey := fmt.Sprintf("-p ARP --arp-op Request --arp-ip-dst %s -j arpreply --arpreply-mac %s", ipAddr.IP.String(), ep.MacAddress.String()) - rulesMap[arpReplyKey] = ebtables.PreRouting - } - - dst := "--ip-dst" - proto := "IPv4" - ipAddress := ipAddr.IP.String() - if ipAddr.IP.To4() == nil { - dst = "--ip6-dst" - proto = "IPv6" - ipAddress = ipAddr.IP.String() + ipv6Mask - } - - dnatMacKey := fmt.Sprintf("-p %s -i %s %s %s -j dnat --to-dst %s --dnat-target ACCEPT", - proto, extIf.Name, dst, ipAddress, ep.MacAddress.String()) - rulesMap[dnatMacKey] = ebtables.PreRouting - } - } - } - } - - return rulesMap -} diff --git a/network/monitor_windows.go b/network/monitor_windows.go deleted file mode 100644 index 8081187e92..0000000000 --- a/network/monitor_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -package network - -import cnms "github.com/Azure/azure-container-networking/cnms/cnmspackage" - -func (nm *networkManager) monitorNetworkState(networkMonitor *cnms.NetworkMonitor) error { - return nil -}