From 4b13351b6a709ceb7e9ca4ef425c8c1372a5aaca Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 20 Jul 2022 22:58:40 +0000 Subject: [PATCH 1/5] iptables snat rules in cns --- cns/NetworkContainerContract.go | 1 + cns/restserver/api_test.go | 6 ++- cns/restserver/internalapi.go | 60 ++++++++++++++++++++++++ cns/restserver/restserver.go | 3 +- cns/restserver/util.go | 6 ++- cns/singletenantcontroller/conversion.go | 1 + cns/types/codes.go | 1 + 7 files changed, 73 insertions(+), 5 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index ae57a204f8..1e50b4edbe 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -73,6 +73,7 @@ const ( // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. type CreateNetworkContainerRequest struct { + HostPrimaryIP string Version string NetworkContainerType string NetworkContainerid string // Mandatory input. diff --git a/cns/restserver/api_test.go b/cns/restserver/api_test.go index 749b0b9e73..f6c7f2d39b 100644 --- a/cns/restserver/api_test.go +++ b/cns/restserver/api_test.go @@ -608,7 +608,8 @@ func TestGetNetworkContainerVersionStatus(t *testing.T) { func createNC( t *testing.T, params createOrUpdateNetworkContainerParams, - expectError bool) { + expectError bool, +) { if err := createOrUpdateNetworkContainerWithParams(t, params); err != nil { t.Errorf("createOrUpdateNetworkContainerWithParams failed Err:%+v", err) t.Fatal(err) @@ -658,7 +659,8 @@ func publishNCViaCNS(t *testing.T, networkID, networkContainerID, createNetworkContainerURL string, - expectError bool) error { + expectError bool, +) error { var ( body bytes.Buffer resp cns.PublishNetworkContainerResponse diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index 763f09d559..3934e71737 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "net" "net/http" "net/http/httptest" "reflect" @@ -20,6 +21,8 @@ import ( "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" + "github.com/Azure/azure-container-networking/iptables" + "github.com/Azure/azure-container-networking/network/networkutils" "github.com/pkg/errors" ) @@ -360,5 +363,62 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req *cns. logger.Errorf(returnMessage) } + returnCode, returnMessage = service.programSNATIPTableRules(req) + if returnCode != 0 { + logger.Errorf(returnMessage) + } + return returnCode } + +func (service *HTTPRestService) programSNATIPTableRules(req *cns.CreateNetworkContainerRequest) (types.ResponseCode, string) { + service.Lock() + defer service.Unlock() + + if service.programmedIPtables { // check if iptables has already been programmed + logger.Printf("[Azure CNS] SNAT IPTables rules already programmed") + return types.Success, "" + } + + ncPrimaryIP, ncIPNet, _ := net.ParseCIDR(req.IPConfiguration.IPSubnet.IPAddress + "/" + fmt.Sprintf("%d", req.IPConfiguration.IPSubnet.PrefixLength)) + + azureDNSUDPMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncIPNet.String(), networkutils.AzureDNS, iptables.UDP, iptables.DNSPort) + azureDNSTCPMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncIPNet.String(), networkutils.AzureDNS, iptables.TCP, iptables.DNSPort) + azureIMDSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncIPNet.String(), networkutils.AzureIMDS, iptables.TCP, iptables.HTTPPort) + + snatPrimaryIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, ncPrimaryIP) + // we need to snat IMDS traffic to node IP, this sets up snat '--to' + snatHostIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, req.HostPrimaryIP) + + // Check if iptables rules exist already + if iptables.RuleExists(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift) && + iptables.RuleExists(iptables.V4, iptables.Nat, iptables.Swift, azureDNSUDPMatch, snatPrimaryIPJump) && + iptables.RuleExists(iptables.V4, iptables.Nat, iptables.Swift, azureDNSTCPMatch, snatPrimaryIPJump) && + iptables.RuleExists(iptables.V4, iptables.Nat, iptables.Swift, azureIMDSMatch, snatHostIPJump) { + + logger.Printf("[Azure CNS] SNAT IPTables rules already programmed") + service.programmedIPtables = true + return types.Success, "" + } + + cmds := []iptables.IPTableEntry{ + iptables.GetCreateChainCmd(iptables.V4, iptables.Nat, iptables.Swift), + iptables.GetAppendIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift), + // add a snat rules to primary NC IP for DNS + iptables.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureDNSUDPMatch, snatPrimaryIPJump), + iptables.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureDNSTCPMatch, snatPrimaryIPJump), + // add a snat rule to node IP for IMDS http traffic + iptables.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureIMDSMatch, snatHostIPJump), + } + + logger.Printf("Adding iptable rules...") + for _, cmd := range cmds { + err := iptables.RunCmd(cmd.Version, cmd.Params) + if err != nil { + return types.FailedToRunIPTableCmd, "failed to run iptable cmd: " + err.Error() + } + logger.Printf("Successfully run iptables rule %v", cmd) + } + + return types.Success, "" +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index d2e3ae8096..cab4c1cb27 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -58,7 +58,8 @@ type HTTPRestService struct { state *httpRestServiceState podsPendingIPAssignment *bounded.TimedSet sync.RWMutex - dncPartitionKey string + dncPartitionKey string + programmedIPtables bool } type GetHTTPServiceDataResponse struct { diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 7711272f3b..8696125019 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -256,7 +256,8 @@ func (service *HTTPRestService) updateIPConfigsStateUntransacted( // If the IP is already added then it will be an idempotent call. Also note, caller will // acquire/release the service lock. func (service *HTTPRestService) addIPConfigStateUntransacted(ncID string, hostVersion int, ipconfigs, - existingSecondaryIPConfigs map[string]cns.SecondaryIPConfig) { + existingSecondaryIPConfigs map[string]cns.SecondaryIPConfig, +) { // add ipconfigs to state for ipID, ipconfig := range ipconfigs { // New secondary IP configs has new NC version however, CNS don't want to override existing IPs'with new @@ -628,7 +629,8 @@ func (service *HTTPRestService) setNetworkStateJoined(networkID string) { // Join Network by calling nmagent func (service *HTTPRestService) joinNetwork( - networkID string) (*http.Response, error, error) { + networkID string, +) (*http.Response, error, error) { var err error joinResponse, joinErr := nmagent.JoinNetwork(networkID) diff --git a/cns/singletenantcontroller/conversion.go b/cns/singletenantcontroller/conversion.go index 413a2d667c..f8f8df32fb 100644 --- a/cns/singletenantcontroller/conversion.go +++ b/cns/singletenantcontroller/conversion.go @@ -56,6 +56,7 @@ func CreateNCRequestFromDynamicNC(nc v1alpha.NetworkContainer) (*cns.CreateNetwo } } return &cns.CreateNetworkContainerRequest{ + HostPrimaryIP: nc.NodeIP, SecondaryIPConfigs: secondaryIPConfigs, NetworkContainerid: nc.ID, NetworkContainerType: cns.Docker, diff --git a/cns/types/codes.go b/cns/types/codes.go index 7705f37925..6d75e13e1b 100644 --- a/cns/types/codes.go +++ b/cns/types/codes.go @@ -38,6 +38,7 @@ const ( NetworkContainerVfpProgramCheckSkipped ResponseCode = 36 NmAgentSupportedApisError ResponseCode = 37 UnsupportedNCVersion ResponseCode = 38 + FailedToRunIPTableCmd ResponseCode = 39 UnexpectedError ResponseCode = 99 ) From 5225b403a788b67123d7850a82cbe23ad2e338a5 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 20 Jul 2022 23:00:57 +0000 Subject: [PATCH 2/5] linting issue --- cns/restserver/internalapi.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index 3934e71737..50ebdbf563 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -371,6 +371,7 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req *cns. return returnCode } +// nolint func (service *HTTPRestService) programSNATIPTableRules(req *cns.CreateNetworkContainerRequest) (types.ResponseCode, string) { service.Lock() defer service.Unlock() From e0aff14103443cbebee938aff4972967a9bd4700 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 20 Jul 2022 23:52:53 +0000 Subject: [PATCH 3/5] add cns config option for running delegated ipam --- cns/configuration/configuration.go | 1 + cns/restserver/internalapi.go | 8 +++++--- cns/service/main.go | 1 + common/config.go | 4 ++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index d9ea331e09..d476f6fc00 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -35,6 +35,7 @@ type CNSConfig struct { WireserverIP string KeyVaultSettings KeyVaultSettings MSISettings MSISettings + DelegatedIPAMUsed bool } type TelemetrySettings struct { diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index 50ebdbf563..5448e5db81 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -363,9 +363,11 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req *cns. logger.Errorf(returnMessage) } - returnCode, returnMessage = service.programSNATIPTableRules(req) - if returnCode != 0 { - logger.Errorf(returnMessage) + if service.Options[common.OptDelegatedIPAM] == true { + returnCode, returnMessage = service.programSNATIPTableRules(req) + if returnCode != 0 { + logger.Errorf(returnMessage) + } } return returnCode diff --git a/cns/service/main.go b/cns/service/main.go index f96f8322b6..1a5ff55fad 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -553,6 +553,7 @@ func main() { httpRestService.SetOption(acn.OptCreateDefaultExtNetworkType, createDefaultExtNetworkType) httpRestService.SetOption(acn.OptHttpConnectionTimeout, httpConnectionTimeout) httpRestService.SetOption(acn.OptHttpResponseHeaderTimeout, httpResponseHeaderTimeout) + httpRestService.SetOption(acn.OptDelegatedIPAM, cnsconfig.DelegatedIPAMUsed) // Create default ext network if commandline option is set if len(strings.TrimSpace(createDefaultExtNetworkType)) > 0 { diff --git a/common/config.go b/common/config.go index 2b8f0bbd9d..8a7e021071 100644 --- a/common/config.go +++ b/common/config.go @@ -87,6 +87,10 @@ const ( OptTelemetry = "telemetry" OptTelemetryAlias = "dt" + // Enable delegated ipam mode + OptDelegatedIPAM = "delegated-ipam" + OptDelegatedIPAMAlias = "d-ipam" + // Enable Telemetry service OptTelemetryService = "telemetry-service" OptTelemetryServiceAlias = "ts" From cfe806a0d001cb980f0f76e315043b9eabe1d30e Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Thu, 21 Jul 2022 00:21:04 +0000 Subject: [PATCH 4/5] change cns config option --- cns/restserver/internalapi.go | 2 +- cns/service/main.go | 2 +- common/config.go | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index 5448e5db81..b144756155 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -363,7 +363,7 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req *cns. logger.Errorf(returnMessage) } - if service.Options[common.OptDelegatedIPAM] == true { + if service.Options[common.OptProgramSNATIPTables] == true { returnCode, returnMessage = service.programSNATIPTableRules(req) if returnCode != 0 { logger.Errorf(returnMessage) diff --git a/cns/service/main.go b/cns/service/main.go index 1a5ff55fad..050ccaaaca 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -553,7 +553,7 @@ func main() { httpRestService.SetOption(acn.OptCreateDefaultExtNetworkType, createDefaultExtNetworkType) httpRestService.SetOption(acn.OptHttpConnectionTimeout, httpConnectionTimeout) httpRestService.SetOption(acn.OptHttpResponseHeaderTimeout, httpResponseHeaderTimeout) - httpRestService.SetOption(acn.OptDelegatedIPAM, cnsconfig.DelegatedIPAMUsed) + httpRestService.SetOption(acn.OptProgramSNATIPTables, cnsconfig.DelegatedIPAMUsed) // Create default ext network if commandline option is set if len(strings.TrimSpace(createDefaultExtNetworkType)) > 0 { diff --git a/common/config.go b/common/config.go index 8a7e021071..996491828a 100644 --- a/common/config.go +++ b/common/config.go @@ -87,9 +87,8 @@ const ( OptTelemetry = "telemetry" OptTelemetryAlias = "dt" - // Enable delegated ipam mode - OptDelegatedIPAM = "delegated-ipam" - OptDelegatedIPAMAlias = "d-ipam" + // Enable CNS to program SNAT iptables rules + OptProgramSNATIPTables = "program-snat-iptables" // Enable Telemetry service OptTelemetryService = "telemetry-service" From 514820e6cde5273ad05dd7ce1575a32b59b6f67a Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Thu, 21 Jul 2022 18:48:03 +0000 Subject: [PATCH 5/5] update var name --- cns/configuration/configuration.go | 2 +- cns/service/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index d476f6fc00..0fd01eaabf 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -35,7 +35,7 @@ type CNSConfig struct { WireserverIP string KeyVaultSettings KeyVaultSettings MSISettings MSISettings - DelegatedIPAMUsed bool + ProgramSNATIPTables bool } type TelemetrySettings struct { diff --git a/cns/service/main.go b/cns/service/main.go index 050ccaaaca..589eaa7ae1 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -553,7 +553,7 @@ func main() { httpRestService.SetOption(acn.OptCreateDefaultExtNetworkType, createDefaultExtNetworkType) httpRestService.SetOption(acn.OptHttpConnectionTimeout, httpConnectionTimeout) httpRestService.SetOption(acn.OptHttpResponseHeaderTimeout, httpResponseHeaderTimeout) - httpRestService.SetOption(acn.OptProgramSNATIPTables, cnsconfig.DelegatedIPAMUsed) + httpRestService.SetOption(acn.OptProgramSNATIPTables, cnsconfig.ProgramSNATIPTables) // Create default ext network if commandline option is set if len(strings.TrimSpace(createDefaultExtNetworkType)) > 0 {