clustermesh: use custom dialer for service resolution

When kvstoremesh is enabled, the agent connects to the local kvstore,
rather to remote ones. Hence, it targets the corresponding service.
Yet, since agents run in host network, service resolution requires that
the DNSPolicy is set to ClusterFirstWithHostNet, introducing a
dependency on CoreDNS. To prevent this requirement, let's configure a
custom dialer responsible for service resolution based on the service
cached information.

Signed-off-by: Marco Iorio <marco.iorio@isovalent.com>

pkg/clustermesh/cell.go

@@ -23,7 +23,7 @@ var Cell = cell.Module(
 	cell.Provide(NewClusterMesh),
 
 	// Convert concrete objects into more restricted interfaces used by clustermesh.
-	cell.ProvidePrivate(func(sc *k8s.ServiceCache) ServiceMerger { return sc }),
+	cell.ProvidePrivate(func(sc *k8s.ServiceCache) (ServiceMerger, k8s.ServiceIPGetter) { return sc, sc }),
 	cell.ProvidePrivate(func(ipcache *ipcache.IPCache) ipcache.IPCacher { return ipcache }),
 	cell.ProvidePrivate(func(mgr nodemanager.NodeManager) (store.Observer, kvstore.ClusterSizeDependantIntervalFunc) {
 		return nodeStore.NewNodeObserver(mgr), mgr.ClusterSizeDependantInterval

pkg/clustermesh/clustermesh.go

@@ -14,6 +14,7 @@ import (
 	"github.com/cilium/cilium/pkg/hive"
 	"github.com/cilium/cilium/pkg/hive/cell"
 	"github.com/cilium/cilium/pkg/ipcache"
+	"github.com/cilium/cilium/pkg/k8s"
 	"github.com/cilium/cilium/pkg/kvstore"
 	"github.com/cilium/cilium/pkg/kvstore/store"
 	"github.com/cilium/cilium/pkg/lock"
@@ -57,6 +58,9 @@ type Configuration struct {
 	// ClusterSizeDependantInterval allows to calculate intervals based on cluster size.
 	ClusterSizeDependantInterval kvstore.ClusterSizeDependantIntervalFunc
 
+	// ServiceIPGetter, if not nil, is used to create a custom dialer for service resolution.
+	ServiceIPGetter k8s.ServiceIPGetter
+
 	Metrics         Metrics
 	InternalMetrics internal.Metrics
 }
@@ -110,6 +114,7 @@ func NewClusterMesh(lifecycle hive.Lifecycle, c Configuration) *ClusterMesh {
 		Config:                       c.Config,
 		ClusterIDName:                c.ClusterIDName,
 		ClusterSizeDependantInterval: c.ClusterSizeDependantInterval,
+		ServiceIPGetter:              c.ServiceIPGetter,
 
 		NewRemoteCluster: cm.newRemoteCluster,
 

pkg/clustermesh/internal/clustermesh.go

@@ -12,6 +12,7 @@ import (
 	"github.com/cilium/cilium/pkg/clustermesh/types"
 	"github.com/cilium/cilium/pkg/controller"
 	"github.com/cilium/cilium/pkg/hive"
+	"github.com/cilium/cilium/pkg/k8s"
 	"github.com/cilium/cilium/pkg/kvstore"
 	"github.com/cilium/cilium/pkg/lock"
 )
@@ -51,6 +52,9 @@ type Configuration struct {
 	// ClusterSizeDependantInterval allows to calculate intervals based on cluster size.
 	ClusterSizeDependantInterval kvstore.ClusterSizeDependantIntervalFunc
 
+	// ServiceIPGetter, if not nil, is used to create a custom dialer for service resolution.
+	ServiceIPGetter k8s.ServiceIPGetter
+
 	// Metrics holds the different clustermesh metrics.
 	Metrics Metrics
 }
@@ -112,6 +116,7 @@ func (cm *ClusterMesh) newRemoteCluster(name, path string) *remoteCluster {
 		name:                         name,
 		configPath:                   path,
 		clusterSizeDependantInterval: cm.conf.ClusterSizeDependantInterval,
+		serviceIPGetter:              cm.conf.ServiceIPGetter,
 
 		changed:     make(chan bool, configNotificationsChannelSize),
 		controllers: controller.NewManager(),

pkg/clustermesh/internal/remote_cluster.go

@@ -13,12 +13,14 @@ import (
 	"github.com/go-openapi/strfmt"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
 
 	"github.com/cilium/cilium/api/v1/models"
 	"github.com/cilium/cilium/pkg/clustermesh/types"
 	cmtypes "github.com/cilium/cilium/pkg/clustermesh/types"
 	cmutils "github.com/cilium/cilium/pkg/clustermesh/utils"
 	"github.com/cilium/cilium/pkg/controller"
+	"github.com/cilium/cilium/pkg/k8s"
 	"github.com/cilium/cilium/pkg/kvstore"
 	"github.com/cilium/cilium/pkg/lock"
 	"github.com/cilium/cilium/pkg/metrics"
@@ -47,6 +49,9 @@ type remoteCluster struct {
 	// clusterSizeDependantInterval allows to calculate intervals based on cluster size.
 	clusterSizeDependantInterval kvstore.ClusterSizeDependantIntervalFunc
 
+	// serviceIPGetter, if not nil, is used to create a custom dialer for service resolution.
+	serviceIPGetter k8s.ServiceIPGetter
+
 	// changed receives an event when the remote cluster configuration has
 	// changed and is closed when the configuration file was removed
 	changed chan bool
@@ -287,10 +292,18 @@ func (rc *remoteCluster) makeEtcdOpts() map[string]string {
 }
 
 func (rc *remoteCluster) makeExtraOpts() kvstore.ExtraOptions {
+	var dialOpts []grpc.DialOption
+	if rc.serviceIPGetter != nil {
+		// Allow to resolve service names without depending on the DNS. This prevents the need
+		// for setting the DNSPolicy to ClusterFirstWithHostNet when running in host network.
+		dialOpts = append(dialOpts, grpc.WithContextDialer(k8s.CreateCustomDialer(rc.serviceIPGetter, rc.getLogger())))
+	}
+
 	return kvstore.ExtraOptions{
 		NoLockQuorumCheck:            true,
 		ClusterName:                  rc.name,
 		ClusterSizeDependantInterval: rc.clusterSizeDependantInterval,
+		DialOption:                   dialOpts,
 	}
 }