From e9bafb6a03e7ca8652c66bb429e3f7b014e90a58 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Thu, 22 Feb 2018 12:57:12 -0800 Subject: [PATCH 1/2] Removed extra blank line --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 9ebce0d28b..3547f54453 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,6 @@ $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT): $(CNIFILES) $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT): $(CNSFILES) go build -v -o $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNS_DIR)/*.go - # Build all binaries in a container. .PHONY: all-binaries-containerized all-binaries-containerized: From 93917e52ff3f2216e63222eb8e8bfebef308d898 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 23 Feb 2018 14:36:12 -0800 Subject: [PATCH 2/2] Restore CNS state and set SNAT rule. Fixed bug in Makefile --- Makefile | 12 ++-- cns/dockerclient/dockerclient.go | 36 ++++-------- cns/restserver/restserver.go | 95 +++++++++++++++++++++++++++++++- 3 files changed, 110 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 3547f54453..9d492834f8 100644 --- a/Makefile +++ b/Makefile @@ -29,12 +29,12 @@ CNIFILES = \ CNSFILES = \ $(wildcard cns/*.go) \ $(wildcard cns/common/*.go) \ - $(wildcard cni/dockerclient/*.go) \ - $(wildcard cni/imdsclient/*.go) \ - $(wildcard cni/ipamclient/*.go) \ - $(wildcard cni/restserver/*.go) \ - $(wildcard cni/routes/*.go) \ - $(wildcard cni/service/*.go) \ + $(wildcard cns/dockerclient/*.go) \ + $(wildcard cns/imdsclient/*.go) \ + $(wildcard cns/ipamclient/*.go) \ + $(wildcard cns/restserver/*.go) \ + $(wildcard cns/routes/*.go) \ + $(wildcard cns/service/*.go) \ $(COREFILES) \ $(CNMFILES) diff --git a/cns/dockerclient/dockerclient.go b/cns/dockerclient/dockerclient.go index 532a267abd..4760d63d6c 100644 --- a/cns/dockerclient/dockerclient.go +++ b/cns/dockerclient/dockerclient.go @@ -8,8 +8,8 @@ import ( "encoding/json" "fmt" "net/http" - "os/exec" + "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/imdsclient" "github.com/Azure/azure-container-networking/log" ) @@ -27,16 +27,6 @@ type DockerClient struct { imdsClient *imdsclient.ImdsClient } -func executeShellCommand(command string) error { - log.Debugf("[Azure-CNS] %s", command) - cmd := exec.Command("sh", "-c", command) - err := cmd.Start() - if err != nil { - return err - } - return cmd.Wait() -} - // NewDockerClient create a new docker client. func NewDockerClient(url string) (*DockerClient, error) { return &DockerClient{ @@ -81,18 +71,13 @@ func (dockerClient *DockerClient) NetworkExists(networkName string) error { } // CreateNetwork creates a network using docker network create. -func (dockerClient *DockerClient) CreateNetwork(networkName string, options map[string]interface{}) error { +func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imdsclient.InterfaceInfo, options map[string]interface{}) error { log.Printf("[Azure CNS] CreateNetwork") enableSnat := true - primaryNic, err := dockerClient.imdsClient.GetPrimaryInterfaceInfoFromHost() - if err != nil { - return err - } - config := &Config{ - Subnet: primaryNic.Subnet, + Subnet: nicInfo.Subnet, } configs := make([]Config, 1) @@ -123,7 +108,7 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, options map[ log.Printf("[Azure CNS] Going to create network with config: %+v", netConfig) netConfigJSON := new(bytes.Buffer) - err = json.NewEncoder(netConfigJSON).Encode(netConfig) + err := json.NewEncoder(netConfigJSON).Encode(netConfig) if err != nil { return err } @@ -150,12 +135,9 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, options map[ } if enableSnat { - cmd := fmt.Sprintf("iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE", - primaryNic.Subnet) - err = executeShellCommand(cmd) + err = common.SetOutboundSNAT(nicInfo.Subnet) if err != nil { - log.Printf("SNAT Iptable rule was not set") - return err + log.Printf("[Azure CNS] Error setting up SNAT outbound rule %v", err) } } @@ -186,7 +168,11 @@ func (dockerClient *DockerClient) DeleteNetwork(networkName string) error { cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE", primaryNic.Subnet) - executeShellCommand(cmd) + err = common.ExecuteShellCommand(cmd) + if err != nil { + log.Printf("[Azure CNS] Error Removing Outbound SNAT rule %v", err) + } + return nil } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 56b4e6b9a9..a5ac5a8705 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -16,6 +16,7 @@ import ( "github.com/Azure/azure-container-networking/cns/ipamclient" "github.com/Azure/azure-container-networking/cns/routes" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/store" ) @@ -32,7 +33,7 @@ type httpRestService struct { ipamClient *ipamclient.IpamClient routingTable *routes.RoutingTable store store.KeyValueStore - state httpRestServiceState + state *httpRestServiceState } // httpRestServiceState contains the state we would like to persist. @@ -40,9 +41,16 @@ type httpRestServiceState struct { Location string NetworkType string Initialized bool + Networks map[string]*networkInfo TimeStamp time.Time } +type networkInfo struct { + NetworkName string + NicInfo *imdsclient.InterfaceInfo + Options map[string]interface{} +} + // HTTPService describes the min API interface that every service should have. type HTTPService interface { common.ServiceAPI @@ -67,6 +75,9 @@ func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) { return nil, err } + serviceState := &httpRestServiceState{} + serviceState.Networks = make(map[string]*networkInfo) + return &httpRestService{ Service: service, store: service.Service.Store, @@ -74,7 +85,9 @@ func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) { imdsClient: imdsClient, ipamClient: ic, routingTable: routingTable, + state: serviceState, }, nil + } // Start starts the CNS listener. @@ -86,6 +99,18 @@ func (service *httpRestService) Start(config *common.ServiceConfig) error { return err } + err = service.restoreState() + if err != nil { + log.Printf("[Azure CNS] Failed to restore state, err:%v.", err) + return err + } + + err = service.restoreNetworkState() + if err != nil { + log.Printf("[Azure CNS] Failed to restore state, err:%v.", err) + return err + } + // Add handlers. listener := service.Listener // default handlers @@ -186,7 +211,14 @@ func (service *httpRestService) createNetwork(w http.ResponseWriter, r *http.Req log.Printf("[Azure CNS] Unable to get routing table from node, %+v.", err.Error()) } - err = dc.CreateNetwork(req.NetworkName, req.Options) + nicInfo, err := service.imdsClient.GetPrimaryInterfaceInfoFromHost() + if err != nil { + returnMessage = fmt.Sprintf("[Azure CNS] Error. CreateNetwork failed %v.", err.Error()) + returnCode = UnexpectedError + break + } + + err = dc.CreateNetwork(req.NetworkName, nicInfo, req.Options) if err != nil { returnMessage = fmt.Sprintf("[Azure CNS] Error. CreateNetwork failed %v.", err.Error()) returnCode = UnexpectedError @@ -197,6 +229,14 @@ func (service *httpRestService) createNetwork(w http.ResponseWriter, r *http.Req log.Printf("[Azure CNS] Unable to restore routing table on node, %+v.", err.Error()) } + networkInfo := &networkInfo{ + NetworkName: req.NetworkName, + NicInfo: nicInfo, + Options: req.Options, + } + + service.state.Networks[req.NetworkName] = networkInfo + case "StandAlone": returnMessage = fmt.Sprintf("[Azure CNS] Error. Underlay network is not supported in StandAlone environment. %v.", err.Error()) returnCode = UnsupportedEnvironment @@ -228,6 +268,10 @@ func (service *httpRestService) createNetwork(w http.ResponseWriter, r *http.Req err = service.Listener.Encode(w, &resp) + if returnCode == 0 { + service.saveState() + } + log.Response(service.Name, resp, err) } @@ -279,6 +323,11 @@ func (service *httpRestService) deleteNetwork(w http.ResponseWriter, r *http.Req err = service.Listener.Encode(w, &resp) + if returnCode == 0 { + delete(service.state.Networks, req.NetworkName) + service.saveState() + } + log.Response(service.Name, resp, err) } @@ -723,3 +772,45 @@ func (service *httpRestService) restoreState() error { log.Printf("[Azure CNS] Restored state, %+v\n", service.state) return nil } + +// restoreNetworkState restores Network state that existed before reboot. +func (service *httpRestService) restoreNetworkState() error { + log.Printf("[Azure CNS] Enter Restoring Network State") + + rebooted := false + + modTime, err := service.store.GetModificationTime() + if err == nil { + log.Printf("[Azure CNS] Store timestamp is %v.", modTime) + + rebootTime, err := platform.GetLastRebootTime() + if err == nil && rebootTime.After(modTime) { + log.Printf("[Azure CNS] reboot time %v mod time %v", rebootTime, modTime) + rebooted = true + } + } + + if rebooted { + for _, nwInfo := range service.state.Networks { + enableSnat := true + + log.Printf("[Azure CNS] Restore nwinfo %v", nwInfo) + + if nwInfo.Options != nil { + if _, ok := nwInfo.Options[dockerclient.OptDisableSnat]; ok { + enableSnat = false + } + } + + if enableSnat { + err := common.SetOutboundSNAT(nwInfo.NicInfo.Subnet) + if err != nil { + log.Printf("[Azure CNS] Error setting up SNAT outbound rule %v", err) + return err + } + } + } + } + + return nil +}