diff --git a/Dockerfile b/Dockerfile index 9c143f48d1..8a31b44543 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,12 @@ -FROM golang:1.6 +FROM golang:latest # Install dependencies. RUN apt-get update && apt-get install -y ebtables RUN go get -d -v golang.org/x/sys/unix -COPY . /go/src/github.com/Azure/Aqua -WORKDIR /go/src/github.com/Azure/Aqua +COPY . /go/src/github.com/Azure/azure-container-networking +WORKDIR /go/src/github.com/Azure/azure-container-networking -RUN go install - -CMD ["/go/bin/Aqua"] +RUN make azure-cnm-plugin +RUN make azure-cni-plugin diff --git a/common/utils.go b/common/utils.go index 451095addb..86b193cfa5 100644 --- a/common/utils.go +++ b/common/utils.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net" "os/exec" + "time" "github.com/Azure/azure-container-networking/log" ) @@ -35,6 +36,26 @@ func LogNetworkInterfaces() { } } +// GetLastRebootTime returns the last time the system rebooted. +func GetLastRebootTime() (time.Time, error) { + // Query last reboot time. + out, err := exec.Command("uptime", "-s").Output() + if err != nil { + log.Printf("Failed to query uptime, err:%v", err) + return time.Time{}, err + } + + // Parse the output. + layout := "2006-01-02 15:04:05" + rebootTime, err := time.Parse(layout, string(out[:len(out)-1])) + if err != nil { + log.Printf("Failed to parse uptime, err:%v", err) + return time.Time{}, err + } + + return rebootTime, nil +} + // ExecuteShellCommand executes a shell command. func ExecuteShellCommand(command string) error { log.Debugf("[shell] %s", command) diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go index a938ee41df..0756022708 100644 --- a/ebtables/ebtables.go +++ b/ebtables/ebtables.go @@ -6,16 +6,18 @@ package ebtables import ( "fmt" "io/ioutil" + "net" "os/exec" "strings" "github.com/Azure/azure-container-networking/log" ) -// Init initializes the ebtables module. -func init() { - installEbtables() -} +const ( + // Ebtables actions. + Append = "-A" + Delete = "-D" +) // InstallEbtables installs the ebtables package. func installEbtables() { @@ -31,64 +33,31 @@ func installEbtables() { } } -// SetupSnatForOutgoingPackets sets up snat -func SetupSnatForOutgoingPackets(interfaceName string, snatAddress string) error { - command := fmt.Sprintf("ebtables -t nat -A POSTROUTING -o %s -j snat --to-source %s --snat-arp", interfaceName, snatAddress) - err := executeShellCommand(command) - if err != nil { - return err - } - return nil -} +// SetSnatForInterface sets a MAC SNAT rule for an interface. +func SetSnatForInterface(interfaceName string, macAddress net.HardwareAddr, action string) error { + command := fmt.Sprintf( + "ebtables -t nat %s POSTROUTING -o %s -j snat --to-src %s --snat-arp", + action, interfaceName, macAddress.String()) -// CleanupSnatForOutgoingPackets cleans up snat -func CleanupSnatForOutgoingPackets(interfaceName string, snatAddress string) error { - command := fmt.Sprintf("ebtables -t nat -D POSTROUTING -o %s -j snat --to-source %s --snat-arp", interfaceName, snatAddress) - err := executeShellCommand(command) - if err != nil { - return err - } - return nil + return executeShellCommand(command) } -// SetupDnatForArpReplies sets up dnat -func SetupDnatForArpReplies(interfaceName string) error { - command := fmt.Sprintf("ebtables -t nat -A PREROUTING -i %s -p arp -j dnat --to-destination ff:ff:ff:ff:ff:ff", interfaceName) - err := executeShellCommand(command) - if err != nil { - return err - } - return nil -} +// SetDnatForArpReplies sets a MAC DNAT rule for ARP replies received on an interface. +func SetDnatForArpReplies(interfaceName string, action string) error { + command := fmt.Sprintf( + "ebtables -t nat %s PREROUTING -p ARP -i %s -j dnat --to-dst ff:ff:ff:ff:ff:ff", + action, interfaceName) -// CleanupDnatForArpReplies cleans up dnat -func CleanupDnatForArpReplies(interfaceName string) error { - command := fmt.Sprintf("ebtables -t nat -D PREROUTING -i %s -p arp -j dnat --to-destination ff:ff:ff:ff:ff:ff", interfaceName) - err := executeShellCommand(command) - if err != nil { - return err - } - return nil + return executeShellCommand(command) } -// SetupDnatBasedOnIPV4Address sets up dnat -func SetupDnatBasedOnIPV4Address(ipv4Address string, macAddress string) error { - command := fmt.Sprintf("ebtables -t nat -A PREROUTING -p IPv4 --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT", ipv4Address, macAddress) - err := executeShellCommand(command) - if err != nil { - return err - } - return nil -} +// SetDnatForIPAddress sets a MAC DNAT rule for an IP address. +func SetDnatForIPAddress(ipAddress net.IP, macAddress net.HardwareAddr, action string) error { + command := fmt.Sprintf( + "ebtables -t nat %s PREROUTING -p IPv4 --ip-dst %s -j dnat --to-dst %s", + action, ipAddress.String(), macAddress.String()) -// RemoveDnatBasedOnIPV4Address cleans up dnat -func RemoveDnatBasedOnIPV4Address(ipv4Address string, macAddress string) error { - command := fmt.Sprintf("ebtables -t nat -D PREROUTING -p IPv4 --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT", ipv4Address, macAddress) - err := executeShellCommand(command) - if err != nil { - return err - } - return nil + return executeShellCommand(command) } func executeShellCommand(command string) error { diff --git a/ipam/manager.go b/ipam/manager.go index a5047bb1d4..6d0d013623 100644 --- a/ipam/manager.go +++ b/ipam/manager.go @@ -5,6 +5,7 @@ package ipam import ( "sync" + "time" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" @@ -19,6 +20,8 @@ const ( // AddressManager manages the set of address spaces and pools allocated to containers. type addressManager struct { + Version string + TimeStamp time.Time AddrSpaces map[string]*addressSpace `json:"AddressSpaces"` store store.KeyValueStore source addressConfigSource @@ -64,6 +67,7 @@ func NewAddressManager() (AddressManager, error) { // Initialize configures address manager. func (am *addressManager) Initialize(config *common.PluginConfig, options map[string]interface{}) error { + am.Version = config.Version am.store = config.Store am.netApi, _ = config.NetApi.(network.NetworkManager) @@ -91,8 +95,20 @@ func (am *addressManager) restore() error { return nil } + // After a reboot, all address resources are implicitly released. + // Ignore the persisted state if it is older than the last reboot time. + modTime, err := am.store.GetModificationTime() + if err == nil { + rebootTime, err := common.GetLastRebootTime() + if err == nil && rebootTime.After(modTime) { + log.Printf("[ipam] Ignoring stale state from store.") + log.Printf("[ipam] Store timestamp %v is older than boot timestamp %v.", modTime, rebootTime) + return nil + } + } + // Read any persisted state. - err := am.store.Read(storeKey, am) + err = am.store.Read(storeKey, am) if err != nil { if err == store.ErrKeyNotFound { return nil @@ -121,6 +137,9 @@ func (am *addressManager) save() error { return nil } + // Update time stamp. + am.TimeStamp = time.Now() + err := am.store.Write(storeKey, am) if err == nil { log.Printf("[ipam] Save succeeded.\n") diff --git a/network/endpoint.go b/network/endpoint.go index 1fd320388b..12618eaf07 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -101,7 +101,7 @@ func (nw *network) newEndpoint(epInfo *EndpointInfo) (*endpoint, error) { // Setup MAC address translation rules for container interface. log.Printf("[net] Setting up MAC address translation rules for endpoint %v.", contIfName) for _, ipAddr := range epInfo.IPAddresses { - err = ebtables.SetupDnatBasedOnIPV4Address(ipAddr.IP.String(), containerIf.HardwareAddr.String()) + err = ebtables.SetDnatForIPAddress(ipAddr.IP, containerIf.HardwareAddr, ebtables.Append) if err != nil { goto cleanup } @@ -240,7 +240,7 @@ func (nw *network) deleteEndpoint(endpointId string) error { // Delete MAC address translation rule. log.Printf("[net] Deleting MAC address translation rules for endpoint %v.", endpointId) for _, ipAddr := range ep.IPAddresses { - err = ebtables.RemoveDnatBasedOnIPV4Address(ipAddr.IP.String(), ep.MacAddress.String()) + err = ebtables.SetDnatForIPAddress(ipAddr.IP, ep.MacAddress, ebtables.Delete) if err != nil { goto cleanup } diff --git a/network/manager.go b/network/manager.go index a0c62e307b..abe391c0ca 100644 --- a/network/manager.go +++ b/network/manager.go @@ -5,6 +5,7 @@ package network import ( "sync" + "time" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" @@ -18,6 +19,8 @@ const ( // NetworkManager manages the set of container networking resources. type networkManager struct { + Version string + TimeStamp time.Time ExternalInterfaces map[string]*externalInterface store store.KeyValueStore sync.Mutex @@ -52,6 +55,7 @@ func NewNetworkManager() (NetworkManager, error) { // Initialize configures network manager. func (nm *networkManager) Initialize(config *common.PluginConfig) error { + nm.Version = config.Version nm.store = config.Store // Restore persisted state. @@ -70,8 +74,20 @@ func (nm *networkManager) restore() error { return nil } + // After a reboot, all address resources are implicitly released. + // Ignore the persisted state if it is older than the last reboot time. + modTime, err := nm.store.GetModificationTime() + if err == nil { + rebootTime, err := common.GetLastRebootTime() + if err == nil && rebootTime.After(modTime) { + log.Printf("[net] Ignoring stale state from store.") + log.Printf("[net] Store timestamp %v is older than boot timestamp %v.", modTime, rebootTime) + return nil + } + } + // Read any persisted state. - err := nm.store.Read(storeKey, nm) + err = nm.store.Read(storeKey, nm) if err != nil { if err == store.ErrKeyNotFound { // Considered successful. @@ -101,6 +117,9 @@ func (nm *networkManager) save() error { return nil } + // Update time stamp. + nm.TimeStamp = time.Now() + err := nm.store.Write(storeKey, nm) if err == nil { log.Printf("[net] Save succeeded.\n") diff --git a/network/network.go b/network/network.go index a82ad9d793..070f121aae 100644 --- a/network/network.go +++ b/network/network.go @@ -172,12 +172,12 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, bri // Setup MAC address translation rules for external interface. log.Printf("[net] Setting up MAC address translation rules for %v.", hostIf.Name) - err = ebtables.SetupSnatForOutgoingPackets(hostIf.Name, hostIf.HardwareAddr.String()) + err = ebtables.SetSnatForInterface(hostIf.Name, hostIf.HardwareAddr, ebtables.Append) if err != nil { goto cleanup } - err = ebtables.SetupDnatForArpReplies(hostIf.Name) + err = ebtables.SetDnatForArpReplies(hostIf.Name, ebtables.Append) if err != nil { goto cleanup } @@ -240,8 +240,8 @@ cleanup: log.Printf("[net] Connecting interface %v failed, err:%v.", extIf.Name, err) // Roll back the changes for the network. - ebtables.CleanupDnatForArpReplies(extIf.Name) - ebtables.CleanupSnatForOutgoingPackets(extIf.Name, extIf.MacAddress.String()) + ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete) + ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete) netlink.DeleteLink(bridgeName) @@ -253,8 +253,8 @@ func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface) log.Printf("[net] Disconnecting interface %v.", extIf.Name) // Cleanup MAC address translation rules. - ebtables.CleanupDnatForArpReplies(extIf.Name) - ebtables.CleanupSnatForOutgoingPackets(extIf.Name, extIf.MacAddress.String()) + ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete) + ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete) // Disconnect external interface from its bridge. err := netlink.SetLinkMaster(extIf.Name, "") diff --git a/store/json.go b/store/json.go index 58b773c8f1..e986a99b71 100644 --- a/store/json.go +++ b/store/json.go @@ -102,7 +102,7 @@ func (kvs *jsonFileStore) Write(key string, value interface{}) error { return kvs.flush() } -// Flush commits in-memory state to backing store. +// Flush commits in-memory state to persistent store. func (kvs *jsonFileStore) Flush() error { kvs.Mutex.Lock() defer kvs.Mutex.Unlock() @@ -193,3 +193,13 @@ func (kvs *jsonFileStore) Unlock() error { return nil } + +// GetModificationTime returns the modification time of the persistent store. +func (kvs *jsonFileStore) GetModificationTime() (time.Time, error) { + info, err := os.Stat(kvs.fileName) + if err != nil { + return time.Time{}, err + } + + return info.ModTime(), nil +} diff --git a/store/store.go b/store/store.go index 75a0e05ea0..dab1a87281 100644 --- a/store/store.go +++ b/store/store.go @@ -5,6 +5,7 @@ package store import ( "fmt" + "time" ) // KeyValueStore represents a persistent store of (key,value) pairs. @@ -14,6 +15,7 @@ type KeyValueStore interface { Flush() error Lock(block bool) error Unlock() error + GetModificationTime() (time.Time, error) } var (