ipsec: New Prometheus metrics for XFRM configs

Documentation/observability/metrics.rst

@@ -343,11 +343,14 @@ Name                                          Labels
 IPSec
 ~~~~~
 
-============================================= ================================================== ========== ========================================================
+============================================= ================================================== ========== ===========================================================
 Name                                          Labels                                             Default    Description
-============================================= ================================================== ========== ========================================================
-``ipsec_xfrm_error``                          ``error``, ``type``                                Enabled    Total number of xfrm errors.
-============================================= ================================================== ========== ========================================================
+============================================= ================================================== ========== ===========================================================
+``ipsec_xfrm_error``                          ``error``, ``type``                                Enabled    Total number of xfrm errors
+``ipsec_keys``                                                                                   Enabled    Number of keys in use
+``ipsec_xfrm_states``                         ``direction``                                      Enabled    Number of XFRM states
+``ipsec_xfrm_policies``                       ``direction``                                      Enabled    Number of XFRM policies
+============================================= ================================================== ========== ===========================================================
 
 eBPF
 ~~~~

cilium-dbg/cmd/encrypt_status.go

@@ -19,17 +19,15 @@ import (
 	"github.com/cilium/cilium/api/v1/models"
 	"github.com/cilium/cilium/pkg/command"
 	"github.com/cilium/cilium/pkg/common"
+	"github.com/cilium/cilium/pkg/common/ipsec"
 )
 
 const (
 	// Cilium uses reqid 1 to tie the IPsec security policies to their matching state
 	ciliumReqId = "1"
 )
 
-type void struct{}
-
 var (
-	voidType    void
 	countErrors int
 	regex       = regexp.MustCompile("oseq[[:blank:]]0[xX]([[:xdigit:]]+)")
 )
@@ -75,25 +73,6 @@ func getXfrmStats(mountPoint string) (int, map[string]int) {
 	return countErrors, errorMap
 }
 
-func countUniqueIPsecKeys() int {
-	// trying to mimic set type data structure
-	// using void data type as struct{} because it does not use any memory
-	keys := make(map[string]void)
-	xfrmStates, err := netlink.XfrmStateList(netlink.FAMILY_ALL)
-	if err != nil {
-		Fatalf("Cannot get xfrm state: %s", err)
-	}
-	for _, v := range xfrmStates {
-		if v.Aead == nil {
-			fmt.Printf("Warning: non-AEAD xfrm state found: %s\n", v.String())
-			continue
-		}
-		keys[string(v.Aead.Key)] = voidType
-	}
-
-	return len(keys)
-}
-
 func extractMaxSequenceNumber(ipOutput string) int64 {
 	maxSeqNum := int64(0)
 	lines := strings.Split(ipOutput, "\n")
@@ -145,7 +124,11 @@ func getEncryptionMode() {
 }
 
 func dumpIPsecStatus() {
-	keys := countUniqueIPsecKeys()
+	xfrmStates, err := netlink.XfrmStateList(netlink.FAMILY_ALL)
+	if err != nil {
+		Fatalf("Cannot get xfrm state: %s", err)
+	}
+	keys := ipsec.CountUniqueIPsecKeys(xfrmStates)
 	oseq := maxSequenceNumber()
 	fmt.Printf("Keys in use: %-26d\n", keys)
 	fmt.Printf("Max Seq. Number: %s\n", oseq)

cilium-dbg/cmd/encrypt_status_test.go

@@ -4,76 +4,13 @@
 package cmd
 
 import (
-	"encoding/hex"
-	"net"
-	"runtime"
-
-	"github.com/cilium/cilium/pkg/testutils"
-
 	. "github.com/cilium/checkmate"
-	"github.com/vishvananda/netlink"
-	"github.com/vishvananda/netns"
 )
 
-type EncryptStatusSuite struct {
-	currentNetNS netns.NsHandle
-}
+type EncryptStatusSuite struct{}
 
 var _ = Suite(&EncryptStatusSuite{})
 
-func (s *EncryptStatusSuite) SetUpSuite(c *C) {
-	testutils.PrivilegedTest(c)
-
-	var err error
-	s.currentNetNS, err = netns.Get()
-	c.Assert(err, IsNil)
-}
-
-func getXfrmState(src string, dst string, spi int, key string) *netlink.XfrmState {
-	k, _ := hex.DecodeString(key)
-	return &netlink.XfrmState{
-		Src:   net.ParseIP(src),
-		Dst:   net.ParseIP(dst),
-		Proto: netlink.XFRM_PROTO_ESP,
-		Mode:  netlink.XFRM_MODE_TUNNEL,
-		Spi:   spi,
-		Aead: &netlink.XfrmStateAlgo{
-			Name:   "rfc4106(gcm(aes))",
-			Key:    k,
-			ICVLen: 64,
-		},
-	}
-}
-
-func (s *EncryptStatusSuite) TestCountUniqueIPsecKeys(c *C) {
-	runtime.LockOSThread()
-	defer runtime.UnlockOSThread()
-
-	ns, err := netns.New()
-	c.Assert(err, IsNil)
-	defer func() { c.Assert(ns.Close(), IsNil) }()
-	defer func() { c.Assert(netns.Set(s.currentNetNS), IsNil) }()
-
-	keys := countUniqueIPsecKeys()
-	c.Assert(keys, Equals, 0)
-
-	err = netlink.XfrmStateAdd(getXfrmState("10.0.0.1", "10.0.0.2", 2, "611d0c8049dd88600ec4f9eded7b1ed540ea607f"))
-	c.Assert(err, IsNil)
-
-	// adding different state with same key
-	err = netlink.XfrmStateAdd(getXfrmState("10.0.0.2", "10.0.0.1", 1, "611d0c8049dd88600ec4f9eded7b1ed540ea607f"))
-	c.Assert(err, IsNil)
-
-	keys = countUniqueIPsecKeys()
-	c.Assert(keys, Equals, 1)
-
-	err = netlink.XfrmStateAdd(getXfrmState("10.0.0.1", "10.0.0.2", 1, "383fa49ea57848c9e85af88a187321f81da54bb6"))
-	c.Assert(err, IsNil)
-
-	keys = countUniqueIPsecKeys()
-	c.Assert(keys, Equals, 2)
-}
-
 const procTestFixtures = "fixtures/proc"
 
 func (s *EncryptStatusSuite) TestGetXfrmStats(c *C) {

pkg/common/ipsec/utils.go

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package ipsec
+
+import (
+	"github.com/vishvananda/netlink"
+)
+
+const (
+	maskStateDir = 0xf00
+	markStateIn  = 0xd00
+	markStateOut = 0xe00
+)
+
+func CountUniqueIPsecKeys(states []netlink.XfrmState) int {
+	keys := make(map[string]bool)
+	for _, s := range states {
+		if s.Aead == nil {
+			continue
+		}
+		keys[string(s.Aead.Key)] = true
+	}
+
+	return len(keys)
+}
+
+func CountXfrmStatesByDir(states []netlink.XfrmState) (int, int) {
+	nbXfrmIn := 0
+	nbXfrmOut := 0
+	for _, s := range states {
+		if s.Mark == nil {
+			continue
+		}
+		switch s.Mark.Value & maskStateDir {
+		case markStateIn:
+			nbXfrmIn++
+		case markStateOut:
+			nbXfrmOut++
+		}
+	}
+	return nbXfrmIn, nbXfrmOut
+}
+
+func CountXfrmPoliciesByDir(states []netlink.XfrmPolicy) (int, int, int) {
+	nbXfrmIn := 0
+	nbXfrmOut := 0
+	nbXfrmFwd := 0
+	for _, p := range states {
+		switch p.Dir {
+		case netlink.XFRM_DIR_IN:
+			nbXfrmIn++
+		case netlink.XFRM_DIR_OUT:
+			nbXfrmOut++
+		case netlink.XFRM_DIR_FWD:
+			nbXfrmFwd++
+		}
+	}
+	return nbXfrmIn, nbXfrmOut, nbXfrmFwd
+}

pkg/common/ipsec/utils_test.go

@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package ipsec
+
+import (
+	"encoding/hex"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/vishvananda/netlink"
+)
+
+func getXfrmState(src string, dst string, spi int, key string, mark uint32) netlink.XfrmState {
+	k, _ := hex.DecodeString(key)
+	return netlink.XfrmState{
+		Src:   net.ParseIP(src),
+		Dst:   net.ParseIP(dst),
+		Proto: netlink.XFRM_PROTO_ESP,
+		Mode:  netlink.XFRM_MODE_TUNNEL,
+		Spi:   spi,
+		Aead: &netlink.XfrmStateAlgo{
+			Name:   "rfc4106(gcm(aes))",
+			Key:    k,
+			ICVLen: 64,
+		},
+		Mark: &netlink.XfrmMark{
+			Value: mark,
+			Mask:  0xffffff00,
+		},
+	}
+}
+
+func getXfrmPolicy(t *testing.T, src string, dst string, dir netlink.Dir) netlink.XfrmPolicy {
+	srcCIDR, err := netlink.ParseIPNet(src)
+	require.NoError(t, err)
+	dstCIDR, err := netlink.ParseIPNet(dst)
+	require.NoError(t, err)
+	return netlink.XfrmPolicy{
+		Dir:   dir,
+		Src:   srcCIDR,
+		Dst:   dstCIDR,
+		Proto: netlink.XFRM_PROTO_ESP,
+	}
+}
+
+func TestCountUniqueIPsecKeys(t *testing.T) {
+	var xfrmStates []netlink.XfrmState
+
+	keys := CountUniqueIPsecKeys(xfrmStates)
+	require.Equal(t, keys, 0)
+
+	xfrmStates = append(xfrmStates, getXfrmState("10.0.0.1", "10.0.0.2", 2, "611d0c8049dd88600ec4f9eded7b1ed540ea607f", 0x12343e00))
+	xfrmStates = append(xfrmStates, getXfrmState("10.0.0.2", "10.0.0.1", 1, "611d0c8049dd88600ec4f9eded7b1ed540ea607f", 0x12343d00))
+
+	keys = CountUniqueIPsecKeys(xfrmStates)
+	require.Equal(t, keys, 1)
+
+	xfrmStates = append(xfrmStates, getXfrmState("10.0.0.1", "10.0.0.2", 1, "383fa49ea57848c9e85af88a187321f81da54bb6", 0x12343e00))
+
+	keys = CountUniqueIPsecKeys(xfrmStates)
+	require.Equal(t, keys, 2)
+}
+
+func TestCountXfrmStatesByDir(t *testing.T) {
+	var xfrmStates []netlink.XfrmState
+
+	nbIn, nbOut := CountXfrmStatesByDir(xfrmStates)
+	require.Equal(t, nbIn, 0)
+	require.Equal(t, nbOut, 0)
+
+	xfrmStates = append(xfrmStates, getXfrmState("10.0.0.1", "10.0.0.2", 2, "611d0c8049dd88600ec4f9eded7b1ed540ea607f", 0x12343e00))
+	xfrmStates = append(xfrmStates, getXfrmState("10.0.0.2", "10.0.0.1", 1, "611d0c8049dd88600ec4f9eded7b1ed540ea607f", 0x12343d00))
+	xfrmStates = append(xfrmStates, getXfrmState("10.0.0.3", "10.0.0.1", 1, "611d0c8049dd88600ec4f9eded7b1ed540ea607f", 0x12343d00))
+
+	nbIn, nbOut = CountXfrmStatesByDir(xfrmStates)
+	require.Equal(t, nbIn, 2)
+	require.Equal(t, nbOut, 1)
+}
+
+func TestCountXfrmPoliciesByDir(t *testing.T) {
+	var xfrmPolicies []netlink.XfrmPolicy
+
+	nbIn, nbOut, nbFwd := CountXfrmPoliciesByDir(xfrmPolicies)
+	require.Equal(t, nbIn, 0)
+	require.Equal(t, nbOut, 0)
+	require.Equal(t, nbFwd, 0)
+
+	xfrmPolicies = append(xfrmPolicies, getXfrmPolicy(t, "10.0.1.0/24", "10.0.0.0/24", netlink.XFRM_DIR_IN))
+	xfrmPolicies = append(xfrmPolicies, getXfrmPolicy(t, "10.0.0.0/24", "10.0.1.0/24", netlink.XFRM_DIR_OUT))
+	xfrmPolicies = append(xfrmPolicies, getXfrmPolicy(t, "10.0.0.0/24", "10.0.2.0/24", netlink.XFRM_DIR_OUT))
+	xfrmPolicies = append(xfrmPolicies, getXfrmPolicy(t, "10.0.0.0/16", "10.0.0.0/16", netlink.XFRM_DIR_FWD))
+
+	nbIn, nbOut, nbFwd = CountXfrmPoliciesByDir(xfrmPolicies)
+	require.Equal(t, nbIn, 1)
+	require.Equal(t, nbOut, 2)
+	require.Equal(t, nbFwd, 1)
+}