From 634dd953f52274fe60a3f47f12b388003fe7af7d Mon Sep 17 00:00:00 2001 From: Omar Abdulaziz Date: Wed, 23 Apr 2025 15:49:21 +0200 Subject: [PATCH] revise mycelium wiring - set mycelium public addresses as peers for private instance, to reduce unnecessary traffic for public nodes in internal communications - export ndmz constant values for external use - init netbase/ifaceutil pkg for common utils between net/netlight --- pkg/netbase/ifaceutil/ifaceutil.go | 66 ++++++++++++++++++++++++++++++ pkg/netlight/resource/mycelium.go | 15 +++++-- pkg/network/ndmz/dualstack.go | 36 ++++++++-------- pkg/network/nr/net_resource.go | 26 +++++------- 4 files changed, 106 insertions(+), 37 deletions(-) create mode 100644 pkg/netbase/ifaceutil/ifaceutil.go diff --git a/pkg/netbase/ifaceutil/ifaceutil.go b/pkg/netbase/ifaceutil/ifaceutil.go new file mode 100644 index 00000000..bedf3ee6 --- /dev/null +++ b/pkg/netbase/ifaceutil/ifaceutil.go @@ -0,0 +1,66 @@ +package ifaceutil + +import ( + "fmt" + "net" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/threefoldtech/zosbase/pkg/network/namespace" + "github.com/vishvananda/netlink" +) + +const ( + myceliumPort = "9651" +) + +// GetIPsForIFace retrieves the IP addresses for a given interface name in a specified network namespace. +// If the namespace name is empty, it retrieves the IP addresses from the host. +func GetIPsForIFace(iface, nsName string) ([]net.IPNet, error) { + getIPs := func() ([]net.IPNet, error) { + var results []net.IPNet + + ln, err := netlink.LinkByName(iface) + if err != nil { + return nil, err + } + + ips, err := netlink.AddrList(ln, netlink.FAMILY_V4) + if err != nil { + return nil, err + } + + for _, ip := range ips { + results = append(results, *ip.IPNet) + } + + return results, nil + } + + if nsName == "" { + return getIPs() + } + + netns, err := namespace.GetByName(nsName) + if err != nil { + return nil, err + } + defer netns.Close() + + var results []net.IPNet + err = netns.Do(func(_ ns.NetNS) error { + var getErr error + results, getErr = getIPs() + return getErr + }) + + return results, err +} + +// BuildMyceliumPeerURLs constructs a list of Mycelium peer URLs from a list of IP networks. +func BuildMyceliumPeerURLs(ips []net.IPNet) []string { + peers := make([]string, len(ips)) + for i, ip := range ips { + peers[i] = fmt.Sprintf("tcp://%s:%s", ip.IP.String(), myceliumPort) + } + return peers +} diff --git a/pkg/netlight/resource/mycelium.go b/pkg/netlight/resource/mycelium.go index 16216513..32f2444c 100644 --- a/pkg/netlight/resource/mycelium.go +++ b/pkg/netlight/resource/mycelium.go @@ -13,13 +13,15 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/pkg/errors" + "github.com/threefoldtech/zosbase/pkg/netbase/ifaceutil" "github.com/threefoldtech/zosbase/pkg/netlight/resource/peers" "github.com/threefoldtech/zosbase/pkg/zinit" ) const ( - myceliumBin = "mycelium" - MyceliumSeedDir = "/tmp/network/mycelium" + myceliumBin = "mycelium" + MyceliumSeedDir = "/tmp/network/mycelium" + myceliumPublicInterface = "zos" myceliumSeedLen = 6 HostMyceliumBr = "br-hmy" @@ -190,7 +192,14 @@ func SetupMycelium(netNS ns.NetNS, mycelium string, seed []byte) error { return fmt.Errorf("failed to create seed file '%s': %w", name, err) } - return ensureMyceliumService(zinit.Default(), &name, list) + ips, err := ifaceutil.GetIPsForIFace(myceliumPublicInterface, "") + if err != nil || len(ips) == 0 { + return fmt.Errorf("failed to get ip for interface 'zos': %w", err) + } + + hostPeers := ifaceutil.BuildMyceliumPeerURLs(ips) + + return ensureMyceliumService(zinit.Default(), &name, hostPeers) } if err := os.WriteFile(filepath.Join(MyceliumSeedDir, HostMyceliumBr), seed, 0444); err != nil { diff --git a/pkg/network/ndmz/dualstack.go b/pkg/network/ndmz/dualstack.go index 1533a711..8cfc3e10 100644 --- a/pkg/network/ndmz/dualstack.go +++ b/pkg/network/ndmz/dualstack.go @@ -37,14 +37,14 @@ const ( //NdmzBridge is the name of the ipv4 routing bridge in the host namespace NdmzBridge = "br-ndmz" - //dmzNamespace name of the dmz namespace - dmzNamespace = "ndmz" + //DmzNamespace name of the dmz namespace + DmzNamespace = "ndmz" ndmzNsMACDerivationSuffix6 = "-ndmz6" ndmzNsMACDerivationSuffix4 = "-ndmz4" - // dmzPub4 ipv4 public interface - dmzPub4 = "npub4" + // DmzPub4 ipv4 public interface + DmzPub4 = "npub4" // dmzPub6 ipv6 public interface dmzPub6 = "npub6" @@ -69,7 +69,7 @@ func New(nodeID string, public *netlink.Bridge) DMZ { } func (d *dmzImpl) Namespace() string { - return dmzNamespace + return DmzNamespace } // Create create the NDMZ network namespace and configure its default routes and addresses @@ -85,9 +85,9 @@ func (d *dmzImpl) Create(ctx context.Context) error { // master will actually be `zos`. In that case, we can't plug the physical // iface, but need to create a veth pair between br-pub and zos. - netNS, err := namespace.GetByName(dmzNamespace) + netNS, err := namespace.GetByName(DmzNamespace) if err != nil { - netNS, err = namespace.Create(dmzNamespace) + netNS, err = namespace.Create(DmzNamespace) if err != nil { return errors.Wrap(err, "failed to create ndmz namespace") } @@ -103,7 +103,7 @@ func (d *dmzImpl) Create(ctx context.Context) error { return errors.Wrapf(err, "ndmz: could not node create pub iface 6") } - if err := createPubIface4(dmzPub4, d.nodeID, netNS); err != nil { + if err := createPubIface4(DmzPub4, d.nodeID, netNS); err != nil { return errors.Wrapf(err, "ndmz: could not create pub iface 4") } @@ -134,7 +134,7 @@ func (d *dmzImpl) Create(ctx context.Context) error { } z := zinit.Default() - dhcpMon := NewDHCPMon(dmzPub4, dmzNamespace, z) + dhcpMon := NewDHCPMon(DmzPub4, DmzNamespace, z) go func() { _ = dhcpMon.Start(ctx) }() @@ -144,7 +144,7 @@ func (d *dmzImpl) Create(ctx context.Context) error { // Delete deletes the NDMZ network namespace func (d *dmzImpl) Delete() error { - netNS, err := namespace.GetByName(dmzNamespace) + netNS, err := namespace.GetByName(DmzNamespace) if err == nil { if err := namespace.Delete(netNS); err != nil { return errors.Wrap(err, "failed to delete ndmz network namespace") @@ -230,7 +230,7 @@ func (d *dmzImpl) AttachNR(networkID, nrNSName string, ipamLeaseDir string) erro func (d *dmzImpl) GetIPFor(inf string) ([]net.IPNet, error) { - netns, err := namespace.GetByName(dmzNamespace) + netns, err := namespace.GetByName(DmzNamespace) if err != nil { return nil, err } @@ -262,7 +262,7 @@ func (d *dmzImpl) GetIPFor(inf string) ([]net.IPNet, error) { func (d *dmzImpl) GetIP(family int) ([]net.IPNet, error) { var links []string if family == netlink.FAMILY_V4 || family == netlink.FAMILY_ALL { - links = append(links, dmzPub4) + links = append(links, DmzPub4) } if family == netlink.FAMILY_V6 || family == netlink.FAMILY_ALL { links = append(links, dmzPub6) @@ -272,7 +272,7 @@ func (d *dmzImpl) GetIP(family int) ([]net.IPNet, error) { return nil, nil } - netns, err := namespace.GetByName(dmzNamespace) + netns, err := namespace.GetByName(DmzNamespace) if err != nil { return nil, err } @@ -305,7 +305,7 @@ func (d *dmzImpl) GetIP(family int) ([]net.IPNet, error) { // Get gateway to given destination ip func (d *dmzImpl) GetDefaultGateway(destination net.IP) (net.IP, error) { - netns, err := namespace.GetByName(dmzNamespace) + netns, err := namespace.GetByName(DmzNamespace) if err != nil { return nil, err } @@ -378,7 +378,7 @@ func (d *dmzImpl) Interfaces() ([]types.IfaceInfo, error) { } // get the ndmz network namespace - ndmz, err := namespace.GetByName(dmzNamespace) + ndmz, err := namespace.GetByName(DmzNamespace) if err != nil { return nil, err } @@ -398,10 +398,10 @@ func waitIP4() error { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - probe, err := dhcp.Probe(ctx, dmzPub4) + probe, err := dhcp.Probe(ctx, DmzPub4) if err != nil { - return errors.Wrapf(err, "error while proping interface '%s'", dmzPub4) + return errors.Wrapf(err, "error while proping interface '%s'", DmzPub4) } if len(probe.IP) != 0 && len(probe.Router) != 0 { return nil @@ -554,7 +554,7 @@ func applyFirewall() error { return errors.Wrap(err, "failed to build nft rule set") } - if err := nft.Apply(&buf, dmzNamespace); err != nil { + if err := nft.Apply(&buf, DmzNamespace); err != nil { return errors.Wrap(err, "failed to apply nft rule set") } diff --git a/pkg/network/nr/net_resource.go b/pkg/network/nr/net_resource.go index 0c78165c..aae10335 100644 --- a/pkg/network/nr/net_resource.go +++ b/pkg/network/nr/net_resource.go @@ -13,10 +13,10 @@ import ( "strings" "time" - "github.com/threefoldtech/zosbase/pkg/environment" "github.com/threefoldtech/zosbase/pkg/gridtypes/zos" "github.com/threefoldtech/zosbase/pkg/network/ifaceutil" "github.com/threefoldtech/zosbase/pkg/network/macvlan" + "github.com/threefoldtech/zosbase/pkg/network/ndmz" "github.com/threefoldtech/zosbase/pkg/network/options" "github.com/threefoldtech/zosbase/pkg/network/tuntap" "github.com/threefoldtech/zosbase/pkg/zinit" @@ -33,6 +33,8 @@ import ( "github.com/threefoldtech/zosbase/pkg/network/namespace" "github.com/threefoldtech/zosbase/pkg/network/wireguard" "github.com/vishvananda/netlink" + + baseifaceutil "github.com/threefoldtech/zosbase/pkg/netbase/ifaceutil" ) const ( @@ -258,11 +260,6 @@ func (nr *NetResource) SetMycelium() (err error) { return nil } - peers, err := environment.GetConfig() - if err != nil { - return errors.Wrap(err, "failed to get public mycelium peer list") - } - config := nr.resource.Mycelium // create the bridge. if err := nr.ensureMyceliumBridge(); err != nil { @@ -311,17 +308,14 @@ func (nr *NetResource) SetMycelium() (err error) { "--peers", } - // first append peers from user input. - // right now this is shadowed by Mycelium config validation - // which does not allow custom peer list. - args = AppendFunc(args, config.Peers, func(mp zos.MyceliumPeer) string { - return string(mp) - }) - - // global peers list - args = append(args, peers.Mycelium.Peers...) + // set mycelium public addresses are the private peers + ips, err := baseifaceutil.GetIPsForIFace(ndmz.DmzPub4, ndmz.DmzNamespace) + if err != nil { + return errors.Wrap(err, "failed to get IPs for npub4") + } - // todo: add custom peers requested by the user + hostPeers := baseifaceutil.BuildMyceliumPeerURLs(ips) + args = append(args, hostPeers...) err = zinit.AddService(name, zinit.InitService{ Exec: strings.Join(args, " "),