Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding support for loading TC eBPF programs on existing ingress and HTB qdisc #359

Merged
merged 1 commit into from
Apr 17, 2024
Merged

Adding support for loading TC eBPF programs on existing ingress and HTB qdisc #359

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
277 changes: 214 additions & 63 deletions kf/kf_unix.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,6 @@ func (b *BPF) LoadTCAttachProgram(ifaceName, direction string) error {
log.Error().Err(err).Msgf("LoadTCAttachProgram - look up network iface %q", ifaceName)
return err
}

if err := b.LoadBPFProgram(ifaceName); err != nil {
return err
}
Expand All @@ -217,8 +216,12 @@ func (b *BPF) LoadTCAttachProgram(ifaceName, direction string) error {
if err != nil {
return fmt.Errorf("could not open rtnetlink socket for interface %s : %w", ifaceName, err)
}

clsactFound := false
htbFound := false
ingressFound := false
var htbHandle uint32
var ingressHandle uint32
var parentHandle uint32
// get all the qdiscs from all interfaces
qdiscs, err := tcgo.Qdisc().Get()
if err != nil {
Expand All @@ -229,12 +232,54 @@ func (b *BPF) LoadTCAttachProgram(ifaceName, direction string) error {
if err != nil {
return fmt.Errorf("could not get interface %s from id %d: %w", ifaceName, qdisc.Ifindex, err)
}
if iface.Name == ifaceName && qdisc.Kind == "clsact" {
clsactFound = true
if iface.Name == ifaceName {
switch qdisc.Kind {
case "clsact":
clsactFound = true
case "htb":
htbFound = true
htbHandle = qdisc.Msg.Handle
case "ingress":
ingressFound = true
ingressHandle = qdisc.Msg.Handle
default:
log.Info().Msgf("Un-supported qdisc kind for interface %s ", ifaceName)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this going to be too verbose? Or is it useful to a user who is inspecting the logs?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe it will be useful for debugging on the hypervisors.

}
}
}

if !clsactFound {
bpfRootProg := b.ProgMapCollection.Programs[b.Program.EntryFunctionName]

var parent uint32
var filter tc.Object

progFD := uint32(bpfRootProg.FD())
// Netlink attribute used in the Linux kernel
bpfFlag := uint32(tc.BpfActDirect)
if clsactFound {
if direction == models.IngressType {
parent = tc.HandleMinIngress
} else if direction == models.EgressType {
parent = tc.HandleMinEgress
}

filter = tc.Object{
Msg: tc.Msg{
Family: unix.AF_UNSPEC,
Ifindex: uint32(iface.Index),
Handle: 0,
Parent: core.BuildHandle(tc.HandleRoot, parent),
Info: 0x300,
},
Attribute: tc.Attribute{
Kind: "bpf",
BPF: &tc.Bpf{
FD: &progFD,
Flags: &bpfFlag,
},
},
}
} else if !clsactFound && !ingressFound && !htbFound {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMHO For clarity, this condition block can be relocated to the end as part of the else statement.

qdisc := tc.Object{
Msg: tc.Msg{
Family: unix.AF_UNSPEC,
Expand All @@ -247,45 +292,62 @@ func (b *BPF) LoadTCAttachProgram(ifaceName, direction string) error {
Kind: "clsact",
},
}

if err := tcgo.Qdisc().Add(&qdisc); err != nil {
log.Info().Msgf("could not assign clsact to %s : %v, its already exists", ifaceName, err)
}
}

bpfRootProg := b.ProgMapCollection.Programs[b.Program.EntryFunctionName]

var parent uint32
if direction == models.IngressType {
parent = tc.HandleMinIngress
} else if direction == models.EgressType {
parent = tc.HandleMinEgress
}
if direction == models.IngressType {
parent = tc.HandleMinIngress
} else if direction == models.EgressType {
parent = tc.HandleMinEgress
}

progFD := uint32(bpfRootProg.FD())
// Netlink attribute used in the Linux kernel
bpfFlag := uint32(tc.BpfActDirect)
filter = tc.Object{
Msg: tc.Msg{
Family: unix.AF_UNSPEC,
Ifindex: uint32(iface.Index),
Handle: 0,
Parent: core.BuildHandle(tc.HandleRoot, parent),
Info: 0x300,
},
Attribute: tc.Attribute{
Kind: "bpf",
BPF: &tc.Bpf{
FD: &progFD,
Flags: &bpfFlag,
},
},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to define tc.Attribute{} outside of the block in order to prevent duplicated code?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As tc.Attribute{} is the field of tc.Object.Attribute, I would prefer to keep the declaration in place for context.

}
} else if !clsactFound && ingressFound && htbFound {
if direction == models.IngressType {
parentHandle = ingressHandle
} else if direction == models.EgressType {
parentHandle = htbHandle
}

filter := tc.Object{
Msg: tc.Msg{
Family: unix.AF_UNSPEC,
Ifindex: uint32(iface.Index),
Handle: 0,
Parent: core.BuildHandle(tc.HandleRoot, parent),
Info: 0x300,
},
Attribute: tc.Attribute{
Kind: "bpf",
BPF: &tc.Bpf{
FD: &progFD,
Flags: &bpfFlag,
// parentNew needs to handle of HTB and ingress 1:, and ffff:
filter = tc.Object{
Msg: tc.Msg{
Family: unix.AF_UNSPEC,
Ifindex: uint32(iface.Index),
Handle: 0,
Parent: parentHandle,
Info: 0x300,
},
Attribute: tc.Attribute{
Kind: "bpf",
BPF: &tc.Bpf{
FD: &progFD,
Flags: &bpfFlag,
},
},
},
}
} else {
log.Info().Msgf("Unable to create qdisc object for interface %s", ifaceName)
}

// Storing Filter handle
b.TCFilter = tcgo.Filter()

// Attaching / Adding as filter
if err := b.TCFilter.Add(&filter); err != nil {
return fmt.Errorf("could not attach filter to interface %s for eBPF program %s : %w", ifaceName, b.Program.Name, err)
Expand All @@ -301,58 +363,147 @@ func (b *BPF) LoadTCAttachProgram(ifaceName, direction string) error {

// UnloadTCProgram - Remove TC filters
func (b *BPF) UnloadTCProgram(ifaceName, direction string) error {

iface, err := net.InterfaceByName(ifaceName)
if err != nil {
log.Error().Err(err).Msgf("UnloadTCProgram - look up network iface %q", ifaceName)
return err
}

bpfRootProg := b.ProgMapCollection.Programs[b.Program.EntryFunctionName]

var parent uint32
if direction == models.IngressType {
parent = tc.HandleMinIngress
} else if direction == models.EgressType {
parent = tc.HandleMinEgress
tcgo, err := tc.Open(&tc.Config{})
if err != nil {
log.Error().Err(err).Msgf("UnloadTCProgram - Unable to tc.Open(&tc.Config{}): %q", ifaceName)
return err
}

tcfilts, err := b.TCFilter.Get(&tc.Msg{
Family: unix.AF_UNSPEC,
Ifindex: uint32(iface.Index),
Handle: 0x0,
Parent: core.BuildHandle(tc.HandleRoot, parent),
})

clsactFound := false
htbFound := false
ingressFound := false
var htbHandle uint32
var ingressHandle uint32
var parentHandle uint32
// get all the qdiscs from all interfaces
qdiscs, err := tcgo.Qdisc().Get()
if err != nil {
log.Warn().Msgf("Could not get filters for interface \"%s\" direction %s ", ifaceName, direction)
return fmt.Errorf("could not get filters for interface %s : %w", ifaceName, err)
}
for _, qdisc := range qdiscs {
iface, err := net.InterfaceByIndex(int(qdisc.Ifindex))
if err != nil {
return fmt.Errorf("could not get interface %s from id %d: %v", ifaceName, qdisc.Ifindex, err)
}
if iface.Name == ifaceName {
switch qdisc.Kind {
case "clsact":
clsactFound = true
case "htb":
htbFound = true
htbHandle = qdisc.Msg.Handle
case "ingress":
ingressFound = true
ingressHandle = qdisc.Msg.Handle
default:
log.Info().Msgf("qdisc kind for %s : %v", ifaceName, err)
}
}
}

progFD := uint32(bpfRootProg.FD())
// Netlink attribute used in the Linux kernel
bpfFlag := uint32(tc.BpfActDirect)
bpfRootProg := b.ProgMapCollection.Programs[b.Program.EntryFunctionName]

var parent uint32
var filter tc.Object

if clsactFound && !ingressFound && !htbFound {
if direction == models.IngressType {
parent = tc.HandleMinIngress
} else if direction == models.EgressType {
parent = tc.HandleMinEgress
}

filter := tc.Object{
Msg: tc.Msg{
tcfilts, err := b.TCFilter.Get(&tc.Msg{
Family: unix.AF_UNSPEC,
Ifindex: uint32(iface.Index),
Handle: 0,
Handle: 0x0,
Parent: core.BuildHandle(tc.HandleRoot, parent),
Info: tcfilts[0].Msg.Info,
},
Attribute: tc.Attribute{
Kind: "bpf",
BPF: &tc.Bpf{
FD: &progFD,
Flags: &bpfFlag,
})

if err != nil {
log.Warn().Msgf("Could not get filters for interface \"%s\" direction %s ", ifaceName, direction)
return fmt.Errorf("could not get filters for interface %s : %v", ifaceName, err)
}

progFD := uint32(bpfRootProg.FD())
// Netlink attribute used in the Linux kernel
bpfFlag := uint32(tc.BpfActDirect)

filter = tc.Object{
Msg: tc.Msg{
Family: unix.AF_UNSPEC,
Ifindex: uint32(iface.Index),
Handle: 0,
Parent: core.BuildHandle(tc.HandleRoot, parent),
Info: tcfilts[0].Msg.Info,
},
Attribute: tc.Attribute{
Kind: "bpf",
BPF: &tc.Bpf{
FD: &progFD,
Flags: &bpfFlag,
},
},
},
}
} else if !clsactFound && ingressFound && htbFound {
if direction == models.EgressType {
parentHandle = htbHandle
// _ = pa("parentNew...1 ", parentNew)
} else if direction == models.IngressType {
parentHandle = ingressHandle
}
tcfilts, err := b.TCFilter.Get(&tc.Msg{
Family: unix.AF_UNSPEC,
Ifindex: uint32(iface.Index),
Handle: 0x0,
Parent: parentHandle,
})

if err != nil {
log.Warn().Msgf("Could not get filters for interface \"%s\" direction %s ", ifaceName, direction)
return fmt.Errorf("could not get filters for interface %s : %v", ifaceName, err)
}

progFD := uint32(bpfRootProg.FD())
// Netlink attribute used in the Linux kernel
bpfFlag := uint32(tc.BpfActDirect)

var tcFilterIndex int
for i, tcfilt := range tcfilts {
// finding the Info field of the relevant BPF filter among all set filters for that qdisc
if tcfilt.Attribute.Kind == "bpf" {
tcFilterIndex = i
}
}
// Add a check for if tcFilterIndex out of bounds
filter = tc.Object{
Msg: tc.Msg{
Family: unix.AF_UNSPEC,
Ifindex: uint32(iface.Index),
Handle: 0,
Parent: parentHandle,
Info: tcfilts[tcFilterIndex].Msg.Info,
},
Attribute: tc.Attribute{
Kind: "bpf",
BPF: &tc.Bpf{
FD: &progFD,
Flags: &bpfFlag,
},
},
}
}

// Detaching / Deleting filter
if err := b.TCFilter.Delete(&filter); err != nil {
return fmt.Errorf("could not dettach tc filter for interface %s : %w", ifaceName, err)
return fmt.Errorf("could not dettach tc filter for interface %s : Direction: %v, parentHandle: %v, Error:%w", ifaceName, direction, parentHandle, err)
}

return nil
Expand Down