Restore CNS State

Makefile

@@ -29,12 +29,12 @@ CNIFILES = \
 CNSFILES = \
 	$(wildcard cns/*.go) \
 	$(wildcard cns/common/*.go) \
-	$(wildcard cni/dockerclient/*.go) \
-	$(wildcard cni/imdsclient/*.go) \
-	$(wildcard cni/ipamclient/*.go) \
-	$(wildcard cni/restserver/*.go) \
-	$(wildcard cni/routes/*.go) \
-	$(wildcard cni/service/*.go) \
+	$(wildcard cns/dockerclient/*.go) \
+	$(wildcard cns/imdsclient/*.go) \
+	$(wildcard cns/ipamclient/*.go) \
+	$(wildcard cns/restserver/*.go) \
+	$(wildcard cns/routes/*.go) \
+	$(wildcard cns/service/*.go) \
 	$(COREFILES) \
 	$(CNMFILES)
 
@@ -113,7 +113,6 @@ $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT): $(CNIFILES)
 $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT): $(CNSFILES)
 	go build -v -o $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNS_DIR)/*.go
 
-
 # Build all binaries in a container.
 .PHONY: all-binaries-containerized
 all-binaries-containerized:

cns/dockerclient/dockerclient.go

@@ -8,8 +8,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"os/exec"
 
+	"github.com/Azure/azure-container-networking/cns/common"
 	"github.com/Azure/azure-container-networking/cns/imdsclient"
 	"github.com/Azure/azure-container-networking/log"
 )
@@ -27,16 +27,6 @@ type DockerClient struct {
 	imdsClient    *imdsclient.ImdsClient
 }
 
-func executeShellCommand(command string) error {
-	log.Debugf("[Azure-CNS] %s", command)
-	cmd := exec.Command("sh", "-c", command)
-	err := cmd.Start()
-	if err != nil {
-		return err
-	}
-	return cmd.Wait()
-}
-
 // NewDockerClient create a new docker client.
 func NewDockerClient(url string) (*DockerClient, error) {
 	return &DockerClient{
@@ -81,18 +71,13 @@ func (dockerClient *DockerClient) NetworkExists(networkName string) error {
 }
 
 // CreateNetwork creates a network using docker network create.
-func (dockerClient *DockerClient) CreateNetwork(networkName string, options map[string]interface{}) error {
+func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imdsclient.InterfaceInfo, options map[string]interface{}) error {
 	log.Printf("[Azure CNS] CreateNetwork")
 
 	enableSnat := true
 
-	primaryNic, err := dockerClient.imdsClient.GetPrimaryInterfaceInfoFromHost()
-	if err != nil {
-		return err
-	}
-
 	config := &Config{
-		Subnet: primaryNic.Subnet,
+		Subnet: nicInfo.Subnet,
 	}
 
 	configs := make([]Config, 1)
@@ -123,7 +108,7 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, options map[
 	log.Printf("[Azure CNS] Going to create network with config: %+v", netConfig)
 
 	netConfigJSON := new(bytes.Buffer)
-	err = json.NewEncoder(netConfigJSON).Encode(netConfig)
+	err := json.NewEncoder(netConfigJSON).Encode(netConfig)
 	if err != nil {
 		return err
 	}
@@ -150,12 +135,9 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, options map[
 	}
 
 	if enableSnat {
-		cmd := fmt.Sprintf("iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE",
-			primaryNic.Subnet)
-		err = executeShellCommand(cmd)
+		err = common.SetOutboundSNAT(nicInfo.Subnet)
 		if err != nil {
-			log.Printf("SNAT Iptable rule was not set")
-			return err
+			log.Printf("[Azure CNS] Error setting up SNAT outbound rule %v", err)
 		}
 	}
 
@@ -186,7 +168,11 @@ func (dockerClient *DockerClient) DeleteNetwork(networkName string) error {
 
 		cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE",
 			primaryNic.Subnet)
-		executeShellCommand(cmd)
+		err = common.ExecuteShellCommand(cmd)
+		if err != nil {
+			log.Printf("[Azure CNS] Error Removing Outbound SNAT rule %v", err)
+		}
+
 		return nil
 	}
 

cns/restserver/restserver.go

@@ -16,6 +16,7 @@ import (
 	"github.com/Azure/azure-container-networking/cns/ipamclient"
 	"github.com/Azure/azure-container-networking/cns/routes"
 	"github.com/Azure/azure-container-networking/log"
+	"github.com/Azure/azure-container-networking/platform"
 	"github.com/Azure/azure-container-networking/store"
 )
 
@@ -32,17 +33,24 @@ type httpRestService struct {
 	ipamClient   *ipamclient.IpamClient
 	routingTable *routes.RoutingTable
 	store        store.KeyValueStore
-	state        httpRestServiceState
+	state        *httpRestServiceState
 }
 
 // httpRestServiceState contains the state we would like to persist.
 type httpRestServiceState struct {
 	Location    string
 	NetworkType string
 	Initialized bool
+	Networks    map[string]*networkInfo
 	TimeStamp   time.Time
 }
 
+type networkInfo struct {
+	NetworkName string
+	NicInfo     *imdsclient.InterfaceInfo
+	Options     map[string]interface{}
+}
+
 // HTTPService describes the min API interface that every service should have.
 type HTTPService interface {
 	common.ServiceAPI
@@ -67,14 +75,19 @@ func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) {
 		return nil, err
 	}
 
+	serviceState := &httpRestServiceState{}
+	serviceState.Networks = make(map[string]*networkInfo)
+
 	return &httpRestService{
 		Service:      service,
 		store:        service.Service.Store,
 		dockerClient: dc,
 		imdsClient:   imdsClient,
 		ipamClient:   ic,
 		routingTable: routingTable,
+		state:        serviceState,
 	}, nil
+
 }
 
 // Start starts the CNS listener.
@@ -86,6 +99,18 @@ func (service *httpRestService) Start(config *common.ServiceConfig) error {
 		return err
 	}
 
+	err = service.restoreState()
+	if err != nil {
+		log.Printf("[Azure CNS]  Failed to restore state, err:%v.", err)
+		return err
+	}
+
+	err = service.restoreNetworkState()
+	if err != nil {
+		log.Printf("[Azure CNS]  Failed to restore state, err:%v.", err)
+		return err
+	}
+
 	// Add handlers.
 	listener := service.Listener
 	// default handlers
@@ -186,7 +211,14 @@ func (service *httpRestService) createNetwork(w http.ResponseWriter, r *http.Req
 								log.Printf("[Azure CNS] Unable to get routing table from node, %+v.", err.Error())
 							}
 
-							err = dc.CreateNetwork(req.NetworkName, req.Options)
+							nicInfo, err := service.imdsClient.GetPrimaryInterfaceInfoFromHost()
+							if err != nil {
+								returnMessage = fmt.Sprintf("[Azure CNS] Error. CreateNetwork failed %v.", err.Error())
+								returnCode = UnexpectedError
+								break
+							}
+
+							err = dc.CreateNetwork(req.NetworkName, nicInfo, req.Options)
 							if err != nil {
 								returnMessage = fmt.Sprintf("[Azure CNS] Error. CreateNetwork failed %v.", err.Error())
 								returnCode = UnexpectedError
@@ -197,6 +229,14 @@ func (service *httpRestService) createNetwork(w http.ResponseWriter, r *http.Req
 								log.Printf("[Azure CNS] Unable to restore routing table on node, %+v.", err.Error())
 							}
 
+							networkInfo := &networkInfo{
+								NetworkName: req.NetworkName,
+								NicInfo:     nicInfo,
+								Options:     req.Options,
+							}
+
+							service.state.Networks[req.NetworkName] = networkInfo
+
 						case "StandAlone":
 							returnMessage = fmt.Sprintf("[Azure CNS] Error. Underlay network is not supported in StandAlone environment. %v.", err.Error())
 							returnCode = UnsupportedEnvironment
@@ -228,6 +268,10 @@ func (service *httpRestService) createNetwork(w http.ResponseWriter, r *http.Req
 
 	err = service.Listener.Encode(w, &resp)
 
+	if returnCode == 0 {
+		service.saveState()
+	}
+
 	log.Response(service.Name, resp, err)
 }
 
@@ -279,6 +323,11 @@ func (service *httpRestService) deleteNetwork(w http.ResponseWriter, r *http.Req
 
 	err = service.Listener.Encode(w, &resp)
 
+	if returnCode == 0 {
+		delete(service.state.Networks, req.NetworkName)
+		service.saveState()
+	}
+
 	log.Response(service.Name, resp, err)
 }
 
@@ -723,3 +772,45 @@ func (service *httpRestService) restoreState() error {
 	log.Printf("[Azure CNS]  Restored state, %+v\n", service.state)
 	return nil
 }
+
+// restoreNetworkState restores Network state that existed before reboot.
+func (service *httpRestService) restoreNetworkState() error {
+	log.Printf("[Azure CNS] Enter Restoring Network State")
+
+	rebooted := false
+
+	modTime, err := service.store.GetModificationTime()
+	if err == nil {
+		log.Printf("[Azure CNS] Store timestamp is %v.", modTime)
+
+		rebootTime, err := platform.GetLastRebootTime()
+		if err == nil && rebootTime.After(modTime) {
+			log.Printf("[Azure CNS] reboot time %v mod time %v", rebootTime, modTime)
+			rebooted = true
+		}
+	}
+
+	if rebooted {
+		for _, nwInfo := range service.state.Networks {
+			enableSnat := true
+
+			log.Printf("[Azure CNS] Restore nwinfo %v", nwInfo)
+
+			if nwInfo.Options != nil {
+				if _, ok := nwInfo.Options[dockerclient.OptDisableSnat]; ok {
+					enableSnat = false
+				}
+			}
+
+			if enableSnat {
+				err := common.SetOutboundSNAT(nwInfo.NicInfo.Subnet)
+				if err != nil {
+					log.Printf("[Azure CNS] Error setting up SNAT outbound rule %v", err)
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}