Skip to content

Commit

Permalink
feat: add routes, routing rules and nftables rules for KubeSpan
Browse files Browse the repository at this point in the history 
This concludes basic KubeSpan implementation.

Most of the code is from #3577 with some fixes and refactoring.

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
Signed-off-by: Seán C McCord <ulexus@gmail.com>
Co-authored-by: Seán C McCord <ulexus@gmail.com>
  • Loading branch information
@smira @Ulexus
smira and Ulexus committed Sep 16, 2021
1 parent ed12379 commit ef36849
Show file tree
Hide file tree
Showing 12 changed files with 789 additions and 16 deletions.
4 changes: 3 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ require (
github.com/gizak/termui/v3 v3.1.0
github.com/google/go-cmp v0.5.6
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/nftables v0.0.0-20210818072602-d553cd2d411d
github.com/google/uuid v1.3.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/hashicorp/go-getter v1.5.8
Expand Down Expand Up @@ -84,6 +85,7 @@ require (
github.com/talos-systems/net v0.3.0
github.com/talos-systems/talos/pkg/machinery v0.0.0-00010101000000-000000000000
github.com/u-root/u-root v7.0.0+incompatible
github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
github.com/vmware-tanzu/sonobuoy v0.53.2
github.com/vmware/govmomi v0.26.1
github.com/vmware/vmw-guestinfo v0.0.0-20200218095840-687661b8bd8e
Expand Down Expand Up @@ -198,6 +200,7 @@ require (
github.com/json-iterator/go v1.1.11 // indirect
github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/klauspost/compress v1.11.13 // indirect
github.com/koneu/natend v0.0.0-20150829182554-ec0926ea948d // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/magiconair/properties v1.8.5 // indirect
Expand Down Expand Up @@ -248,7 +251,6 @@ require (
github.com/subosito/gotenv v1.2.0 // indirect
github.com/u-root/uio v0.0.0-20210528114334-82958018845c // indirect
github.com/ulikunitz/xz v0.5.8 // indirect
github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 // indirect
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect
Expand Down
7 changes: 7 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/nftables v0.0.0-20210818072602-d553cd2d411d h1:thhkCYZ9yhpaJwc0EAAnuXNOf4LK78iq864di/H7AEI=
github.com/google/nftables v0.0.0-20210818072602-d553cd2d411d/go.mod h1:cfspEyr/Ap+JDIITA+N9a0ernqG0qZ4W1aqMRgDZa1g=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
Expand Down Expand Up @@ -772,6 +774,8 @@ github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4=
github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/koneu/natend v0.0.0-20150829182554-ec0926ea948d h1:MFX8DxRnKMY/2M3H61iSsVbo/n3h0MWGmWNN1UViOU0=
github.com/koneu/natend v0.0.0-20150829182554-ec0926ea948d/go.mod h1:QHb4k4cr1fQikUahfcRVPcEXiUgFsdIstGqlurL0XL4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
Expand Down Expand Up @@ -838,6 +842,7 @@ github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lk
github.com/mdlayher/genetlink v1.0.0 h1:OoHN1OdyEIkScEmRgxLEe2M9U8ClMytqA5niynLtfj0=
github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc=
github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA=
github.com/mdlayher/netlink v0.0.0-20191009155606-de872b0d824b/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M=
github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY=
github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o=
Expand Down Expand Up @@ -1384,6 +1389,7 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
Expand Down Expand Up @@ -1487,6 +1493,7 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191029155521-f43be2a4598c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down
129 changes: 129 additions & 0 deletions internal/app/machined/pkg/controllers/kubespan/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ const DefaultPeerReconcileInterval = 30 * time.Second
// ManagerController sets up Wireguard networking based on KubeSpan configuration, watches and updates peer statuses.
type ManagerController struct {
WireguardClientFactory WireguardClientFactory
RulesManagerFactory RulesManagerFactory
NfTablesManagerFactory NfTablesManagerFactory
PeerReconcileInterval time.Duration
}

Expand All @@ -52,6 +54,12 @@ type WireguardClient interface {
Close() error
}

// RulesManagerFactory allows mocking RulesManager.
type RulesManagerFactory func(targetTable, internalMark int) RulesManager

// NfTablesManagerFactory allows mocking NfTablesManager.
type NfTablesManagerFactory func(externalMark, internalMark uint32) NfTablesManager

// Inputs implements controller.Controller interface.
func (ctrl *ManagerController) Inputs() []controller.Input {
return []controller.Input{
Expand Down Expand Up @@ -112,6 +120,14 @@ func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, lo
}
}

if ctrl.RulesManagerFactory == nil {
ctrl.RulesManagerFactory = NewRulesManager
}

if ctrl.NfTablesManagerFactory == nil {
ctrl.NfTablesManagerFactory = NewNfTablesManager
}

if ctrl.PeerReconcileInterval == 0 {
ctrl.PeerReconcileInterval = DefaultPeerReconcileInterval
}
Expand All @@ -123,6 +139,26 @@ func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, lo

defer wgClient.Close() //nolint:errcheck

var rulesMgr RulesManager

defer func() {
if rulesMgr != nil {
if err := rulesMgr.Cleanup(); err != nil {
logger.Error("failed cleaning up routing rules", zap.Error(err))
}
}
}()

var nfTablesMgr NfTablesManager

defer func() {
if nfTablesMgr != nil {
if err := nfTablesMgr.Cleanup(); err != nil {
logger.Error("failed cleaning up nftables rules", zap.Error(err))
}
}
}()

for {
var updateSpecs bool

Expand Down Expand Up @@ -151,6 +187,22 @@ func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, lo
return err
}

if rulesMgr != nil {
if err = rulesMgr.Cleanup(); err != nil {
logger.Error("failed cleaning up routing rules", zap.Error(err))
}

rulesMgr = nil
}

if nfTablesMgr != nil {
if err = nfTablesMgr.Cleanup(); err != nil {
logger.Error("failed cleaning up nftables rules", zap.Error(err))
}

nfTablesMgr = nil
}

continue
}

Expand All @@ -159,6 +211,18 @@ func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, lo
tickerC = ticker.C
}

if rulesMgr == nil {
rulesMgr = ctrl.RulesManagerFactory(constants.KubeSpanDefaultRoutingTable, constants.KubeSpanDefaultForceFirewallMark)

if err = rulesMgr.Install(); err != nil {
return fmt.Errorf("failed setting up routing rules: %w", err)
}
}

if nfTablesMgr == nil {
nfTablesMgr = ctrl.NfTablesManagerFactory(constants.KubeSpanDefaultFirewallMark, constants.KubeSpanDefaultForceFirewallMark)
}

cfgSpec := cfg.(*kubespan.Config).TypedSpec()

localIdentity, err := r.Get(ctx, resource.NewMetadata(kubespan.NamespaceName, kubespan.IdentityType, kubespan.LocalIdentity, resource.VersionUndefined))
Expand Down Expand Up @@ -272,6 +336,20 @@ func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, lo
})
}

// build full allowedIPs set
var allowedIPsBuilder netaddr.IPSetBuilder

for _, peerSpec := range peerSpecs {
for _, prefix := range peerSpec.AllowedIPs {
allowedIPsBuilder.AddPrefix(prefix)
}
}

allowedIPsSet, err := allowedIPsBuilder.IPSet()
if err != nil {
return fmt.Errorf("failed building allowed IPs set: %w", err)
}

// update peer statuses
for pubKey, peerStatus := range peerStatuses {
peerStatus := peerStatus
Expand Down Expand Up @@ -317,6 +395,53 @@ func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, lo
return fmt.Errorf("error modifying address: %w", err)
}

for _, spec := range []network.RouteSpecSpec{
{
Family: nethelpers.FamilyInet4,
Destination: netaddr.IPPrefix{},
Source: netaddr.IP{},
Gateway: netaddr.IP{},
OutLinkName: constants.KubeSpanLinkName,
Table: nethelpers.RoutingTable(constants.KubeSpanDefaultRoutingTable),
Priority: 1,
Scope: nethelpers.ScopeGlobal,
Type: nethelpers.TypeUnicast,
Flags: 0,
Protocol: nethelpers.ProtocolStatic,
ConfigLayer: network.ConfigOperator,
},
{
Family: nethelpers.FamilyInet6,
Destination: netaddr.IPPrefix{},
Source: netaddr.IP{},
Gateway: netaddr.IP{},
OutLinkName: constants.KubeSpanLinkName,
Table: nethelpers.RoutingTable(constants.KubeSpanDefaultRoutingTable),
Priority: 1,
Scope: nethelpers.ScopeGlobal,
Type: nethelpers.TypeUnicast,
Flags: 0,
Protocol: nethelpers.ProtocolStatic,
ConfigLayer: network.ConfigOperator,
},
} {
spec := spec

if err = r.Modify(ctx,
network.NewRouteSpec(
network.ConfigNamespaceName,
network.LayeredID(network.ConfigOperator, network.RouteID(spec.Table, spec.Family, spec.Destination, spec.Gateway, spec.Priority)),
),
func(r resource.Resource) error {
*r.(*network.RouteSpec).TypedSpec() = spec

return nil
},
); err != nil {
return fmt.Errorf("error modifying route spec: %w", err)
}
}

if err = r.Modify(ctx,
network.NewLinkSpec(
network.ConfigNamespaceName,
Expand Down Expand Up @@ -345,6 +470,10 @@ func (ctrl *ManagerController) Run(ctx context.Context, r controller.Runtime, lo
); err != nil {
return fmt.Errorf("error modifying link spec: %w", err)
}

if err = nfTablesMgr.Update(allowedIPsSet); err != nil {
return fmt.Errorf("failed updating nftables: %w", err)
}
}
}

Expand Down
77 changes: 77 additions & 0 deletions internal/app/machined/pkg/controllers/kubespan/manager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package kubespan_test

import (
"fmt"
"net"
"os"
"sync"
Expand Down Expand Up @@ -72,13 +73,55 @@ func (mock *mockWireguardClient) Close() error {
return nil
}

type mockRulesManager struct{}

func (mock mockRulesManager) Install() error {
return nil
}

func (mock mockRulesManager) Cleanup() error {
return nil
}

type mockNftablesManager struct {
mu sync.Mutex
ipSet *netaddr.IPSet
}

func (mock *mockNftablesManager) Update(ipSet *netaddr.IPSet) error {
mock.mu.Lock()
defer mock.mu.Unlock()

mock.ipSet = ipSet

return nil
}

func (mock *mockNftablesManager) Cleanup() error {
return nil
}

func (mock *mockNftablesManager) IPSet() *netaddr.IPSet {
mock.mu.Lock()
defer mock.mu.Unlock()

return mock.ipSet
}

func (suite *ManagerSuite) TestReconcile() {
mockWireguard := &mockWireguardClient{}
mockNfTables := &mockNftablesManager{}

suite.Require().NoError(suite.runtime.RegisterController(&kubespanctrl.ManagerController{
WireguardClientFactory: func() (kubespanctrl.WireguardClient, error) {
return mockWireguard, nil
},
RulesManagerFactory: func(_, _ int) kubespanctrl.RulesManager {
return mockRulesManager{}
},
NfTablesManagerFactory: func(_, _ uint32) kubespanctrl.NfTablesManager {
return mockNfTables
},
PeerReconcileInterval: time.Second,
}))

Expand Down Expand Up @@ -150,6 +193,21 @@ func (suite *ManagerSuite) TestReconcile() {
),
))

suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
suite.assertResourceIDs(
resource.NewMetadata(
network.ConfigNamespaceName,
network.RouteSpecType,
"",
resource.VersionUndefined,
),
[]resource.ID{
network.LayeredID(network.ConfigOperator, network.RouteID(constants.KubeSpanDefaultRoutingTable, nethelpers.FamilyInet4, netaddr.IPPrefix{}, netaddr.IP{}, 1)),
network.LayeredID(network.ConfigOperator, network.RouteID(constants.KubeSpanDefaultRoutingTable, nethelpers.FamilyInet6, netaddr.IPPrefix{}, netaddr.IP{}, 1)),
},
),
))

suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
suite.assertNoResourceType(
resource.NewMetadata(kubespan.NamespaceName, kubespan.PeerStatusType, "", resource.VersionUndefined),
Expand Down Expand Up @@ -233,6 +291,25 @@ func (suite *ManagerSuite) TestReconcile() {
))
}

suite.Assert().NoError(retry.Constant(3*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
func() error {
ipSet := mockNfTables.IPSet()

if ipSet == nil {
return retry.ExpectedErrorf("ipset is nil")
}

ranges := fmt.Sprintf("%v", ipSet.Ranges())
expected := "[10.244.1.0-10.244.2.255]"

if ranges != expected {
return retry.ExpectedErrorf("ranges %s != expected %s", ranges, expected)
}

return nil
},
))

// report up status via wireguard mock
mockWireguard.update(&wgtypes.Device{
Peers: []wgtypes.Peer{
Expand Down

0 comments on commit ef36849

Please sign in to comment.