ipsec: Reinitialize IPsec for new devices in ENI mode

pkg/datapath/loader/base.go

@@ -193,23 +193,25 @@ func (l *Loader) reinitializeIPSec(ctx context.Context) error {
 		return nil
 	}
 
+	l.ipsecMu.Lock()
+	defer l.ipsecMu.Unlock()
+
 	interfaces := option.Config.EncryptInterface
 	if option.Config.IPAM == ipamOption.IPAMENI {
 		// IPAMENI mode supports multiple network facing interfaces that
 		// will all need Encrypt logic applied in order to decrypt any
 		// received encrypted packets. This logic will attach to all
-		// !veth devices. Only use if user has not configured interfaces.
-		if len(interfaces) == 0 {
-			if links, err := netlink.LinkList(); err == nil {
-				for _, link := range links {
-					isVirtual, err := ethtool.IsVirtualDriver(link.Attrs().Name)
-					if err == nil && !isVirtual {
-						interfaces = append(interfaces, link.Attrs().Name)
-					}
+		// !veth devices.
+		interfaces = nil
+		if links, err := netlink.LinkList(); err == nil {
+			for _, link := range links {
+				isVirtual, err := ethtool.IsVirtualDriver(link.Attrs().Name)
+				if err == nil && !isVirtual {
+					interfaces = append(interfaces, link.Attrs().Name)
 				}
 			}
-			option.Config.EncryptInterface = interfaces
 		}
+		option.Config.EncryptInterface = interfaces
 	}
 
 	// No interfaces is valid in tunnel disabled case
@@ -497,8 +499,19 @@ func (l *Loader) Reinitialize(ctx context.Context, o datapath.BaseProgramOwner,
 		log.WithError(err).Fatal("C and Go structs alignment check failed")
 	}
 
-	if err := l.reinitializeIPSec(ctx); err != nil {
-		return err
+	if option.Config.EnableIPSec {
+		if err := compileNetwork(ctx); err != nil {
+			log.WithError(err).Fatal("failed to compile encryption programs")
+		}
+
+		if err := l.reinitializeIPSec(ctx); err != nil {
+			return err
+		}
+
+		if firstInitialization {
+			// Start a background worker to reinitialize IPsec if links change.
+			l.reloadIPSecOnLinkChanges()
+		}
 	}
 
 	if err := l.reinitializeOverlay(ctx, encapProto); err != nil {

pkg/datapath/loader/loader.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cilium/cilium/pkg/defaults"
 	"github.com/cilium/cilium/pkg/elf"
 	iputil "github.com/cilium/cilium/pkg/ip"
+	"github.com/cilium/cilium/pkg/lock"
 	"github.com/cilium/cilium/pkg/logging"
 	"github.com/cilium/cilium/pkg/logging/logfields"
 	"github.com/cilium/cilium/pkg/maps/callsmap"
@@ -67,6 +68,8 @@ type Loader struct {
 
 	// templateCache is the cache of pre-compiled datapaths.
 	templateCache *objectCache
+
+	ipsecMu lock.Mutex // guards reinitializeIPSec
 }
 
 // NewLoader returns a new loader.
@@ -344,15 +347,14 @@ func (l *Loader) reloadDatapath(ctx context.Context, ep datapath.Endpoint, dirs
 }
 
 func (l *Loader) replaceNetworkDatapath(ctx context.Context, interfaces []string) error {
-	if err := compileNetwork(ctx); err != nil {
-		log.WithError(err).Fatal("failed to compile encryption programs")
-	}
 	progs := []progDefinition{{progName: symbolFromNetwork, direction: dirIngress}}
 	for _, iface := range option.Config.EncryptInterface {
 		finalize, err := replaceDatapath(ctx, iface, networkObj, progs, "")
 		if err != nil {
 			log.WithField(logfields.Interface, iface).WithError(err).Fatal("Load encryption network failed")
 		}
+		log.WithField(logfields.Interface, iface).Info("Encryption network program (re)loaded")
+
 		// Defer map removal until all interfaces' progs have been replaced.
 		defer finalize()
 	}

pkg/datapath/loader/netlink.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"time"
 
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
@@ -17,6 +18,8 @@ import (
 
 	"github.com/cilium/cilium/pkg/bpf"
 	"github.com/cilium/cilium/pkg/defaults"
+	"github.com/cilium/cilium/pkg/inctimer"
+	ipamOption "github.com/cilium/cilium/pkg/ipam/option"
 	"github.com/cilium/cilium/pkg/mac"
 	"github.com/cilium/cilium/pkg/option"
 	"github.com/cilium/cilium/pkg/sysctl"
@@ -329,3 +332,74 @@ func SetupBaseDevice(mtu int) (netlink.Link, netlink.Link, error) {
 
 	return linkHost, linkNet, nil
 }
+
+// reloadIPSecOnLinkChanges subscribes to link changes to detect newly added devices
+// and reinitializes IPsec on changes. Only in effect for ENI mode in which we expect
+// new devices at runtime.
+func (l *Loader) reloadIPSecOnLinkChanges() {
+	// settleDuration is the amount of time to wait for further link updates
+	// before proceeding with reinitialization. This avoids back-to-back
+	// reinitialization when multiple link changes are made at once.
+	const settleDuration = 1 * time.Second
+
+	if !option.Config.EnableIPSec || option.Config.IPAM != ipamOption.IPAMENI {
+		return
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	updates := make(chan netlink.LinkUpdate)
+
+	if err := netlink.LinkSubscribe(updates, ctx.Done()); err != nil {
+		log.WithError(err).Fatal("Failed to subscribe for link changes")
+	}
+
+	go func() {
+		defer cancel()
+
+		timer, stop := inctimer.New()
+		defer stop()
+
+		// If updates arrive during settle duration a single element
+		// is sent to this channel and we reinitialize right away
+		// without waiting for further updates.
+		trigger := make(chan struct{}, 1)
+
+		for {
+			// Wait for first update or trigger before reinitializing.
+			select {
+			case _, ok := <-updates:
+				if !ok {
+					return
+				}
+			case <-trigger:
+			}
+
+			log.Info("Reinitializing IPsec due to link changes")
+			err := l.reinitializeIPSec(ctx)
+			if err != nil {
+				// We may fail if links have been removed during the reload. In this case
+				// the updates channel will have queued updates which will retrigger the
+				// reinitialization.
+				log.WithError(err).Warn("Failed to reinitialize IPsec after device change")
+			}
+
+			// Avoid reinitializing repeatedly in short period of time
+			// by draining further updates for 'settleDuration'.
+			settled := timer.After(settleDuration)
+		settleLoop:
+			for {
+				select {
+				case <-settled:
+					break settleLoop
+				case <-updates:
+					select {
+					case trigger <- struct{}{}:
+					default:
+					}
+					break settleLoop
+				}
+
+			}
+		}
+	}()
+}