From cc785b83775bdc6fdcd7af3163a754bce82b8011 Mon Sep 17 00:00:00 2001 From: Arko Dasgupta Date: Mon, 8 Jul 2019 13:52:31 -0700 Subject: [PATCH] Support dockerd and system restarts for ipvlan and macvlan networks This commit carries forward the work done in https://github.com/docker/libnetwork/pull/2295 and fixes two things 1. Allows macvlan and ipvlan to be restored properly after dockerd or the system is restarted 2. Makes sure the refcount for the configOnly network is not incremented for the above case so this network can be deleted after all the associated ConfigFrom networks are deleted Addresses: https://github.com/docker/libnetwork/issues/1743 Signed-off-by: Arko Dasgupta --- controller.go | 17 +++++--- drivers/ipvlan/ipvlan_network.go | 65 +++++++++++++++++++---------- drivers/ipvlan/ipvlan_setup.go | 18 ++++++++ drivers/ipvlan/ipvlan_store.go | 11 ++++- drivers/macvlan/macvlan_network.go | 66 ++++++++++++++++++++---------- drivers/macvlan/macvlan_setup.go | 18 ++++++++ drivers/macvlan/macvlan_store.go | 12 +++++- 7 files changed, 156 insertions(+), 51 deletions(-) diff --git a/controller.go b/controller.go index ff7b36947a..dc33ded09e 100644 --- a/controller.go +++ b/controller.go @@ -706,9 +706,10 @@ const overlayDSROptionString = "dsr" // are network specific and modeled in a generic way. func (c *controller) NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error) { var ( - cap *driverapi.Capability - err error - t *network + cap *driverapi.Capability + err error + t *network + skipCfgEpCount bool ) if id != "" { @@ -803,7 +804,7 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ... } network.generic[netlabel.Internal] = network.internal defer func() { - if err == nil { + if err == nil && !skipCfgEpCount { if err := t.getEpCnt().IncEndpointCnt(); err != nil { logrus.Warnf("Failed to update reference count for configuration network %q on creation of network %q: %v", t.Name(), network.Name(), err) @@ -824,7 +825,13 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ... err = c.addNetwork(network) if err != nil { - return nil, err + if strings.Contains(err.Error(), "restoring existing network") { + // This error can be ignored and set this boolean + // value to skip a refcount increment for configOnly networks + skipCfgEpCount = true + } else { + return nil, err + } } defer func() { if err != nil { diff --git a/drivers/ipvlan/ipvlan_network.go b/drivers/ipvlan/ipvlan_network.go index 39980fb316..a96d86934a 100644 --- a/drivers/ipvlan/ipvlan_network.go +++ b/drivers/ipvlan/ipvlan_network.go @@ -60,10 +60,14 @@ func (d *driver) CreateNetwork(nid string, option map[string]interface{}, nInfo // empty parent and --internal are handled the same. Set here to update k/v config.Internal = true } - err = d.createNetwork(config) + foundExisting, err := d.createNetwork(config) if err != nil { return err } + + if foundExisting { + return types.InternalMaskableErrorf("restoring existing network %s", config.ID) + } // update persistent db, rollback on fail err = d.storeUpdate(config) if err != nil { @@ -76,22 +80,34 @@ func (d *driver) CreateNetwork(nid string, option map[string]interface{}, nInfo } // createNetwork is used by new network callbacks and persistent network cache -func (d *driver) createNetwork(config *configuration) error { +func (d *driver) createNetwork(config *configuration) (bool, error) { + foundExisting := false networkList := d.getNetworks() for _, nw := range networkList { if config.Parent == nw.config.Parent { - return fmt.Errorf("network %s is already using parent interface %s", - getDummyName(stringid.TruncateID(nw.config.ID)), config.Parent) + if config.ID != nw.config.ID { + return false, fmt.Errorf("network %s is already using parent interface %s", + getDummyName(stringid.TruncateID(nw.config.ID)), config.Parent) + } + logrus.Debugf("Create Network for the same ID %s\n", config.ID) + foundExisting = true + break } } if !parentExists(config.Parent) { // if the --internal flag is set, create a dummy link if config.Internal { - err := createDummyLink(config.Parent, getDummyName(stringid.TruncateID(config.ID))) - if err != nil { - return err + if !dummyLinkExists(getDummyName(stringid.TruncateID(config.ID))) { + err := createDummyLink(config.Parent, getDummyName(stringid.TruncateID(config.ID))) + if err != nil { + return false, err + } + config.CreatedSlaveLink = true + + } else { + logrus.Debugf("Dummy Link %s for ipvlan already exists", getDummyName(stringid.TruncateID(config.ID))) } - config.CreatedSlaveLink = true + // notify the user in logs they have limited communications if config.Parent == getDummyName(stringid.TruncateID(config.ID)) { logrus.Debugf("Empty -o parent= and --internal flags limit communications to other containers inside of network: %s", @@ -100,24 +116,31 @@ func (d *driver) createNetwork(config *configuration) error { } else { // if the subinterface parent_iface.vlan_id checks do not pass, return err. // a valid example is 'eth0.10' for a parent iface 'eth0' with a vlan id '10' - err := createVlanLink(config.Parent) - if err != nil { - return err + if !vlanLinkExists(config.Parent) { + err := createVlanLink(config.Parent) + if err != nil { + return false, err + } + // if driver created the networks slave link, record it for future deletion + config.CreatedSlaveLink = true + } else { + logrus.Debugf("Parent Sub Interface %s already Exists NetID %s", config.Parent, config.ID) } - // if driver created the networks slave link, record it for future deletion - config.CreatedSlaveLink = true + } } - n := &network{ - id: config.ID, - driver: d, - endpoints: endpointTable{}, - config: config, + if !foundExisting { + n := &network{ + id: config.ID, + driver: d, + endpoints: endpointTable{}, + config: config, + } + // add the network + d.addNetwork(n) } - // add the *network - d.addNetwork(n) - return nil + return foundExisting, nil } // DeleteNetwork the network for the specified driver type diff --git a/drivers/ipvlan/ipvlan_setup.go b/drivers/ipvlan/ipvlan_setup.go index da8d8faeb8..d7edcbedd1 100644 --- a/drivers/ipvlan/ipvlan_setup.go +++ b/drivers/ipvlan/ipvlan_setup.go @@ -70,6 +70,15 @@ func parentExists(ifaceStr string) bool { return true } +// vlanLinkExists checks if specified vlan link exists in the default namespace +func vlanLinkExists(linkStr string) bool { + _, err := ns.NlHandle().LinkByName(linkStr) + if err != nil { + return false + } + return true +} + // createVlanLink parses sub-interfaces and vlan id for creation func createVlanLink(parentName string) error { if strings.Contains(parentName, ".") { @@ -156,6 +165,15 @@ func parseVlan(linkName string) (string, int, error) { return parent, vidInt, nil } +// dummyLinkExists checks if dummylink exists in the default namespace +func dummyLinkExists(dummyName string) bool { + _, err := ns.NlHandle().LinkByName(dummyName) + if err != nil { + return false + } + return true +} + // createDummyLink creates a dummy0 parent link func createDummyLink(dummyName, truncNetID string) error { // create a parent interface since one was not specified diff --git a/drivers/ipvlan/ipvlan_store.go b/drivers/ipvlan/ipvlan_store.go index 72eb3fc4ff..cf9d324292 100644 --- a/drivers/ipvlan/ipvlan_store.go +++ b/drivers/ipvlan/ipvlan_store.go @@ -55,7 +55,14 @@ func (d *driver) initStore(option map[string]interface{}) error { return types.InternalErrorf("ipvlan driver failed to initialize data store: %v", err) } - return d.populateNetworks() + err = d.populateNetworks() + if err != nil { + return err + } + err = d.populateEndpoints() + if err != nil { + return err + } } return nil @@ -73,7 +80,7 @@ func (d *driver) populateNetworks() error { } for _, kvo := range kvol { config := kvo.(*configuration) - if err = d.createNetwork(config); err != nil { + if _, err = d.createNetwork(config); err != nil { logrus.Warnf("could not create ipvlan network for id %s from persistent state", config.ID) } } diff --git a/drivers/macvlan/macvlan_network.go b/drivers/macvlan/macvlan_network.go index 8c515bb2f4..78bc086fc9 100644 --- a/drivers/macvlan/macvlan_network.go +++ b/drivers/macvlan/macvlan_network.go @@ -64,10 +64,15 @@ func (d *driver) CreateNetwork(nid string, option map[string]interface{}, nInfo // empty parent and --internal are handled the same. Set here to update k/v config.Internal = true } - err = d.createNetwork(config) + foundExisting, err := d.createNetwork(config) if err != nil { return err } + + if foundExisting { + return types.InternalMaskableErrorf("restoring existing network %s", config.ID) + } + // update persistent db, rollback on fail err = d.storeUpdate(config) if err != nil { @@ -80,22 +85,32 @@ func (d *driver) CreateNetwork(nid string, option map[string]interface{}, nInfo } // createNetwork is used by new network callbacks and persistent network cache -func (d *driver) createNetwork(config *configuration) error { +func (d *driver) createNetwork(config *configuration) (bool, error) { + foundExisting := false networkList := d.getNetworks() for _, nw := range networkList { if config.Parent == nw.config.Parent { - return fmt.Errorf("network %s is already using parent interface %s", - getDummyName(stringid.TruncateID(nw.config.ID)), config.Parent) + if config.ID != nw.config.ID { + return false, fmt.Errorf("network %s is already using parent interface %s", + getDummyName(stringid.TruncateID(nw.config.ID)), config.Parent) + } + logrus.Debugf("Create Network for the same ID %s\n", config.ID) + foundExisting = true + break } } if !parentExists(config.Parent) { // if the --internal flag is set, create a dummy link if config.Internal { - err := createDummyLink(config.Parent, getDummyName(stringid.TruncateID(config.ID))) - if err != nil { - return err + if !dummyLinkExists(getDummyName(stringid.TruncateID(config.ID))) { + err := createDummyLink(config.Parent, getDummyName(stringid.TruncateID(config.ID))) + if err != nil { + return false, err + } + config.CreatedSlaveLink = true + } else { + logrus.Debugf("Dummy Link %s for Mac Vlan already exists", getDummyName(stringid.TruncateID(config.ID))) } - config.CreatedSlaveLink = true // notify the user in logs they have limited communications if config.Parent == getDummyName(stringid.TruncateID(config.ID)) { logrus.Debugf("Empty -o parent= and --internal flags limit communications to other containers inside of network: %s", @@ -104,24 +119,33 @@ func (d *driver) createNetwork(config *configuration) error { } else { // if the subinterface parent_iface.vlan_id checks do not pass, return err. // a valid example is 'eth0.10' for a parent iface 'eth0' with a vlan id '10' - err := createVlanLink(config.Parent) - if err != nil { - return err + + if !vlanLinkExists(config.Parent) { + // if the subinterface parent_iface.vlan_id checks do not pass, return err. + // a valid example is 'eth0.10' for a parent iface 'eth0' with a vlan id '10' + err := createVlanLink(config.Parent) + if err != nil { + return false, err + } + // if driver created the networks slave link, record it for future deletion + config.CreatedSlaveLink = true + } else { + logrus.Debugf("Parent Sub Interface %s already Exists NetID %s", config.Parent, config.ID) } - // if driver created the networks slave link, record it for future deletion - config.CreatedSlaveLink = true } } - n := &network{ - id: config.ID, - driver: d, - endpoints: endpointTable{}, - config: config, + if !foundExisting { + n := &network{ + id: config.ID, + driver: d, + endpoints: endpointTable{}, + config: config, + } + // add the network + d.addNetwork(n) } - // add the *network - d.addNetwork(n) - return nil + return foundExisting, nil } // DeleteNetwork deletes the network for the specified driver type diff --git a/drivers/macvlan/macvlan_setup.go b/drivers/macvlan/macvlan_setup.go index fc33ebd707..2d760c2daf 100644 --- a/drivers/macvlan/macvlan_setup.go +++ b/drivers/macvlan/macvlan_setup.go @@ -74,6 +74,15 @@ func parentExists(ifaceStr string) bool { return true } +// vlanLinkExists checks if specified vlan link exists in the default namespace +func vlanLinkExists(linkStr string) bool { + _, err := ns.NlHandle().LinkByName(linkStr) + if err != nil { + return false + } + return true +} + // createVlanLink parses sub-interfaces and vlan id for creation func createVlanLink(parentName string) error { if strings.Contains(parentName, ".") { @@ -160,6 +169,15 @@ func parseVlan(linkName string) (string, int, error) { return parent, vidInt, nil } +// dummyLinkExists checks if dummylink exists in the default namespace +func dummyLinkExists(dummyName string) bool { + _, err := ns.NlHandle().LinkByName(dummyName) + if err != nil { + return false + } + return true +} + // createDummyLink creates a dummy0 parent link func createDummyLink(dummyName, truncNetID string) error { // create a parent interface since one was not specified diff --git a/drivers/macvlan/macvlan_store.go b/drivers/macvlan/macvlan_store.go index 8683cacd02..184e3da957 100644 --- a/drivers/macvlan/macvlan_store.go +++ b/drivers/macvlan/macvlan_store.go @@ -55,7 +55,15 @@ func (d *driver) initStore(option map[string]interface{}) error { return types.InternalErrorf("macvlan driver failed to initialize data store: %v", err) } - return d.populateNetworks() + err = d.populateNetworks() + if err != nil { + return err + } + err = d.populateEndpoints() + if err != nil { + return err + } + } return nil @@ -73,7 +81,7 @@ func (d *driver) populateNetworks() error { } for _, kvo := range kvol { config := kvo.(*configuration) - if err = d.createNetwork(config); err != nil { + if _, err = d.createNetwork(config); err != nil { logrus.Warnf("Could not create macvlan network for id %s from persistent state", config.ID) } }