fqdn: dnsproxy: fix forwarding of the original security identity for TCP

In case of TCP this is not enough to do net.Dial + setsockopt(SO_MARK), as in
this case TCP SYN will have a wrong identity, e.g.:

    Policy verdict log: flow 0x7a95a133 local EP ID 393, remote ID 14616, proto 6, egress, action redirect, match L3-L4, 10.244.1.122:42437 -> 10.244.1.120:53 tcp SYN
    Policy verdict log: flow 0x907eaa19 local EP ID 458, remote ID host, proto 6, ingress, action allow, match L3-Only, 172.19.0.2:56276 -> 10.244.1.120:53 tcp SYN

Here the second message has wrong identity (host). We still allow the traffic,
as the origin is local host and the coredns is running on the same host, but
this will not work for a remote host if ingress policy doesn't allow
remote-node identity.) To fix this we need to pass a Control parameter to Dial,
so that setsockopt(2) is called before the connect(2). With such a change we
now see the correct identity in case of TCP:

    Policy verdict log: flow 0xeb7902a9 local EP ID 393, remote ID 14616, proto 6, egress, action redirect, match L3-L4, 10.244.1.122:36661 -> 10.244.1.120:53 tcp SYN
    Policy verdict log: flow 0x4efbc5a0 local EP ID 458, remote ID 41903, proto 6, ingress, action allow, match L3-L4, 172.19.0.2:40508 -> 10.244.1.120:53 tcp SYN

Fixes: 44c1def ("fqdn: dnsproxy: forward the original security identity")

Signed-off-by: Anton Protopopov <aspsk@isovalent.com>

pkg/fqdn/dnsproxy/proxy.go

@@ -9,11 +9,11 @@ import (
 	"fmt"
 	"math"
 	"net"
-	"os"
 	"regexp"
 	"strconv"
 	"strings"
 	"sync/atomic"
+	"syscall"
 	"time"
 
 	"github.com/miekg/dns"
@@ -732,28 +732,10 @@ func (p *DNSProxy) CheckAllowed(endpointID uint64, destPort uint16, destID ident
 	return false, nil
 }
 
-func configureConnection(conn *net.Conn, secId identity.NumericIdentity) error {
-	var file *os.File
-	var err error
-
-	switch l4conn := (*conn).(type) {
-	case *net.UDPConn:
-		if file, err = l4conn.File(); err != nil {
-			return fmt.Errorf("can't get file from %v: %w", l4conn, err)
-		}
-	case *net.TCPConn:
-		if file, err = l4conn.File(); err != nil {
-			return fmt.Errorf("can't get file from %v: %w", l4conn, err)
-		}
-	default:
-		return fmt.Errorf("unsupported type %T", l4conn)
-	}
-
-	defer file.Close()
-
+func setSoMark(fd int, secId identity.NumericIdentity) error {
 	mark := linux_defaults.MagicMarkIdentity
 	mark |= int(uint32(secId&0xFFFF)<<16 | uint32((secId&0xFF0000)>>16))
-	err = unix.SetsockoptInt(int(file.Fd()), unix.SOL_SOCKET, unix.SO_MARK, mark)
+	err := unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_MARK, mark)
 	if err != nil {
 		return fmt.Errorf("error setting SO_MARK: %w", err)
 	}
@@ -897,6 +879,19 @@ func (p *DNSProxy) ServeDNS(w dns.ResponseWriter, request *dns.Msg) {
 	stat.ProcessingTime.End(true)
 	stat.UpstreamTime.Start()
 
+	dialer := net.Dialer{
+		Timeout: 2 * time.Second,
+		Control: func(network, address string, c syscall.RawConn) error {
+			var soerr error
+			if err := c.Control(func(su uintptr) {
+				soerr = setSoMark(int(su), ep.GetIdentity())
+			}); err != nil {
+				return err
+			}
+			return soerr
+		}}
+	client.Dialer = &dialer
+
 	conn, err := client.Dial(targetServerAddr)
 	if err != nil {
 		err := fmt.Errorf("failed to dial connection to %v: %w", targetServerAddr, err)
@@ -908,15 +903,6 @@ func (p *DNSProxy) ServeDNS(w dns.ResponseWriter, request *dns.Msg) {
 	}
 	defer conn.Close()
 
-	if err = configureConnection(&conn.Conn, ep.GetIdentity()); err != nil {
-		err := fmt.Errorf("failed to set socket options: %w", err)
-		stat.Err = err
-		scopedLog.WithError(err).Error("Failed to configure connection to the upstream DNS server, cannot service DNS request")
-		p.NotifyOnDNSMsg(time.Now(), ep, epIPPort, targetServerID, targetServerAddr, request, protocol, false, &stat)
-		p.sendRefused(scopedLog, w, request)
-		return
-	}
-
 	request.Id = dns.Id() // force a random new ID for this request
 	response, _, err := client.ExchangeWithConn(request, conn)
 	stat.UpstreamTime.End(err == nil)