fqdn: Make maximum number of IPs per restored rule configurable

Only count the number of IPs for each FQDN selector/rule when storing
rules for restoration, rather than ignoring later rules on a port
after previous rules have hit the maximum number of IPs.

Make the maximum number of IPs per restored rule configurable with the
new option '--tofqdns-max-ips-per-restored-rule' (default 1000).

Signed-off-by: Jarno Rajahalme <jarno@covalent.io>

daemon/cmd/daemon_main.go

@@ -786,6 +786,9 @@ func init() {
 	flags.Int(option.ToFQDNsMaxIPsPerHost, defaults.ToFQDNsMaxIPsPerHost, "Maximum number of IPs to maintain per FQDN name for each endpoint")
 	option.BindEnv(option.ToFQDNsMaxIPsPerHost)
 
+	flags.Int(option.ToFQDNsMaxIPsPerRestoredRule, defaults.ToFQDNsMaxIPsPerRestoredRule, "Maximum number of IPs to maintain for each restored rule")
+	option.BindEnv(option.ToFQDNsMaxIPsPerRestoredRule)
+
 	flags.Int(option.ToFQDNsMaxDeferredConnectionDeletes, defaults.ToFQDNsMaxDeferredConnectionDeletes, "Maximum number of IPs to retain for expired DNS lookups with still-active connections")
 	option.BindEnv(option.ToFQDNsMaxDeferredConnectionDeletes)
 

daemon/cmd/fqdn.go

@@ -299,7 +299,9 @@ func (d *Daemon) bootstrapFQDN(possibleEndpoints map[uint16]*endpoint.Endpoint,
 	if err != nil {
 		return err
 	}
-	proxy.DefaultDNSProxy, err = dnsproxy.StartDNSProxy("", port, option.Config.ToFQDNsEnableDNSCompression, d.lookupEPByIP, d.LookupSecIDByIP, d.lookupIPsBySecID, d.notifyOnDNSMsg)
+	proxy.DefaultDNSProxy, err = dnsproxy.StartDNSProxy("", port, option.Config.ToFQDNsEnableDNSCompression,
+		option.Config.ToFQDNsMaxIPsPerRestoredRule, d.lookupEPByIP, d.LookupSecIDByIP, d.lookupIPsBySecID,
+		d.notifyOnDNSMsg)
 	if err == nil {
 		// Increase the ProxyPort reference count so that it will never get released.
 		err = d.l7Proxy.SetProxyPort(listenerName, proxy.DefaultDNSProxy.BindPort)

pkg/defaults/defaults.go

@@ -102,6 +102,10 @@ const (
 	// for each FQDN name in an endpoint's FQDN cache
 	ToFQDNsMaxIPsPerHost = 50
 
+	// ToFQDNsMaxIPsPerRestoredRule defines the maximum number of IPs to maintain
+	// for each FQDN selector in endpoint's restored ToFQDN rules.
+	ToFQDNsMaxIPsPerRestoredRule = 1000
+
 	// ToFQDNsMaxDeferredConnectionDeletes Maximum number of IPs to retain for
 	// expired DNS lookups with still-active connections
 	ToFQDNsMaxDeferredConnectionDeletes = 10000

pkg/fqdn/dnsproxy/proxy.go

@@ -119,6 +119,10 @@ type DNSProxy struct {
 	// helpers.go but is modified during testing.
 	lookupTargetDNSServer func(w dns.ResponseWriter) (serverIP net.IP, serverPort uint16, addrStr string, err error)
 
+	// maxIPsPerRestoredRule is the maximum number of IPs to maintain for each
+	// restored ToFQDNs rule.
+	maxIPsPerRestoredRule int
+
 	// this mutex protects variables below this point
 	lock.Mutex
 
@@ -182,32 +186,30 @@ func (p *DNSProxy) GetRules(endpointID uint16) restore.DNSRules {
 
 	restored := make(restore.DNSRules)
 	for port, entries := range p.allowed[uint64(endpointID)] {
-		count := 0
 		var ipRules restore.IPRules
 		for cs, regex := range entries {
 			var IPs map[string]struct{}
 			if !cs.IsWildcard() {
 				IPs = make(map[string]struct{})
+				count := 0
 			Loop:
 				for _, nid := range cs.GetSelections() {
 					for _, ip := range p.LookupIPsBySecID(nid) {
 						IPs[ip] = struct{}{}
 						count++
-						if count > 1000 {
+						if count > p.maxIPsPerRestoredRule {
 							log.WithFields(logrus.Fields{
 								logfields.EndpointID:            endpointID,
 								logfields.Port:                  port,
 								logfields.EndpointLabelSelector: cs,
-							}).Warning("Too many DNS rules or destinations for a port, skipping the rest")
+								logfields.Limit:                 p.maxIPsPerRestoredRule,
+							}).Warning("Too many IPs for a rule, skipping the rest")
 							break Loop
 						}
 					}
 				}
 			}
 			ipRules = append(ipRules, restore.IPRule{IPs: IPs, Re: restore.RuleRegex{Regexp: regex}})
-			if count > 1000 {
-				break
-			}
 		}
 		restored[port] = ipRules
 	}
@@ -355,7 +357,7 @@ func (proxyStat *ProxyRequestContext) IsTimeout() bool {
 // notifyFunc will be called with DNS response data that is returned to a
 // requesting endpoint. Note that denied requests will not trigger this
 // callback.
-func StartDNSProxy(address string, port uint16, enableDNSCompression bool, lookupEPFunc LookupEndpointIDByIPFunc, lookupSecIDFunc LookupSecIDByIPFunc, lookupIPsFunc LookupIPsBySecIDFunc, notifyFunc NotifyOnDNSMsgFunc) (*DNSProxy, error) {
+func StartDNSProxy(address string, port uint16, enableDNSCompression bool, maxRestoreIPs int, lookupEPFunc LookupEndpointIDByIPFunc, lookupSecIDFunc LookupSecIDByIPFunc, lookupIPsFunc LookupIPsBySecIDFunc, notifyFunc NotifyOnDNSMsgFunc) (*DNSProxy, error) {
 	if port == 0 {
 		log.Debug("DNS Proxy port is configured to 0. A random port will be assigned by the OS.")
 	}
@@ -374,6 +376,7 @@ func StartDNSProxy(address string, port uint16, enableDNSCompression bool, looku
 		restored:                 make(perEPRestored),
 		restoredEPs:              make(restoredEPs),
 		EnableDNSCompression:     enableDNSCompression,
+		maxIPsPerRestoredRule:    maxRestoreIPs,
 	}
 	atomic.StoreInt32(&p.rejectReply, dns.RcodeRefused)
 

pkg/fqdn/dnsproxy/proxy_test.go

@@ -193,7 +193,7 @@ func (s *DNSProxyTestSuite) SetUpTest(c *C) {
 	s.dnsServer = setupServer(c)
 	c.Assert(s.dnsServer, Not(IsNil), Commentf("unable to setup DNS server"))
 
-	proxy, err := StartDNSProxy("", 0, true, // any address, any port, enable compression
+	proxy, err := StartDNSProxy("", 0, true, 1000, // any address, any port, enable compression, max 1000 restore IPs
 		// LookupEPByIP
 		func(ip net.IP) (*endpoint.Endpoint, error) {
 			if s.restoring {

pkg/logging/logfields/logfields.go

@@ -444,6 +444,9 @@ const (
 	// performed
 	Reason = "reason"
 
+	// Limit is a numerical limit that has been exceeded
+	Limit = "limit"
+
 	// Debug is a boolean value for whether debug is set or not.
 	Debug = "debug"
 

pkg/option/config.go

@@ -394,6 +394,10 @@ const (
 	// for each FQDN name in an endpoint's FQDN cache
 	ToFQDNsMaxIPsPerHost = "tofqdns-endpoint-max-ip-per-hostname"
 
+	// ToFQDNMaxIPsPerRestoredRule defines the maximum number of IPs to maintain
+	// for each FQDN selector in endpoint's restored ToFQDN rules
+	ToFQDNsMaxIPsPerRestoredRule = "tofqdns-max-ips-per-restored-rule"
+
 	// ToFQDNsMaxDeferredConnectionDeletes defines the maximum number of IPs to
 	// retain for expired DNS lookups with still-active connections"
 	ToFQDNsMaxDeferredConnectionDeletes = "tofqdns-max-deferred-connection-deletes"
@@ -908,6 +912,7 @@ var HelpFlagSections = []FlagsSection{
 		Flags: []string{
 			FQDNRejectResponseCode,
 			ToFQDNsMaxIPsPerHost,
+			ToFQDNsMaxIPsPerRestoredRule,
 			ToFQDNsMinTTL,
 			ToFQDNsPreCache,
 			ToFQDNsProxyPort,
@@ -1640,6 +1645,10 @@ type DaemonConfig struct {
 	// for each FQDN name in an endpoint's FQDN cache
 	ToFQDNsMaxIPsPerHost int
 
+	// ToFQDNMaxIPsPerRestoredRule defines the maximum number of IPs to maintain
+	// for each FQDN selector in endpoint's restored ToFQDN rules
+	ToFQDNsMaxIPsPerRestoredRule int
+
 	// ToFQDNsMaxIPsPerHost defines the maximum number of IPs to retain for
 	// expired DNS lookups with still-active connections
 	ToFQDNsMaxDeferredConnectionDeletes int
@@ -2052,6 +2061,7 @@ var (
 		EnableL7Proxy:                defaults.EnableL7Proxy,
 		EndpointStatus:               make(map[string]struct{}),
 		ToFQDNsMaxIPsPerHost:         defaults.ToFQDNsMaxIPsPerHost,
+		ToFQDNsMaxIPsPerRestoredRule: defaults.ToFQDNsMaxIPsPerRestoredRule,
 		KVstorePeriodicSync:          defaults.KVstorePeriodicSync,
 		KVstoreConnectivityTimeout:   defaults.KVstoreConnectivityTimeout,
 		IPAllocationTimeout:          defaults.IPAllocationTimeout,
@@ -2579,6 +2589,7 @@ func (c *DaemonConfig) Populate() {
 
 	// toFQDNs options
 	c.ToFQDNsMaxIPsPerHost = viper.GetInt(ToFQDNsMaxIPsPerHost)
+	c.ToFQDNsMaxIPsPerRestoredRule = viper.GetInt(ToFQDNsMaxIPsPerRestoredRule)
 	if maxZombies := viper.GetInt(ToFQDNsMaxDeferredConnectionDeletes); maxZombies >= 0 {
 		c.ToFQDNsMaxDeferredConnectionDeletes = viper.GetInt(ToFQDNsMaxDeferredConnectionDeletes)
 	} else {