diff --git a/vendor.conf b/vendor.conf index 96cc6df711354..686a4130826c0 100644 --- a/vendor.conf +++ b/vendor.conf @@ -135,13 +135,13 @@ google.golang.org/genproto 3f1135a288c9a07e340ae8ba4cc6 github.com/containerd/containerd 7cfa023d95d37076d5ab035003d4839f4b6ba791 https://github.com/moby/containerd.git # master (v1.5.0-dev) + patch for CVE-2021-41190 and CVE-2022-24769 github.com/containerd/fifo 0724c46b320cf96bb172a0550c19a4b1fca4dacb github.com/containerd/continuity efbc4488d8fe1bdc16bde3b2d2990d9b3a899165 -github.com/containerd/cgroups 0b889c03f102012f1d93a97ddd3ef71cd6f4f510 +github.com/containerd/cgroups b9de8a2212026c07cec67baf3323f1fc0121e048 # v1.0.1 github.com/containerd/console 5d7e1412f07b502a01029ea20e20e0d2be31fa7c # v1.0.1 github.com/containerd/go-runc 16b287bc67d069a60fa48db15f330b790b74365b github.com/containerd/typeurl cd3ce7159eae562a4f60ceff37dada11a939d247 # v1.0.1 github.com/containerd/ttrpc bfba540dc45464586c106b1f31c8547933c1eb41 # v1.0.2 github.com/gogo/googleapis 01e0f9cca9b92166042241267ee2a5cdf5cff46c # v1.3.2 -github.com/cilium/ebpf 1c8d4c9ef7759622653a1d319284a44652333b28 +github.com/cilium/ebpf 32458a752e0bcbc670691625f84e7ef6aef5655d # v0.4.0 # cluster github.com/docker/swarmkit cf1e0f8d95ee37ea1f9a5f8a54c73c1e0fa58a54 # bump_20.10 diff --git a/vendor/github.com/cilium/ebpf/README.md b/vendor/github.com/cilium/ebpf/README.md new file mode 100644 index 0000000000000..7f504d3346770 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/README.md @@ -0,0 +1,54 @@ +# eBPF + +[![PkgGoDev](https://pkg.go.dev/badge/github.com/cilium/ebpf)](https://pkg.go.dev/github.com/cilium/ebpf) + +eBPF is a pure Go library that provides utilities for loading, compiling, and +debugging eBPF programs. It has minimal external dependencies and is intended to +be used in long running processes. + +* [asm](https://pkg.go.dev/github.com/cilium/ebpf/asm) contains a basic + assembler +* [link](https://pkg.go.dev/github.com/cilium/ebpf/link) allows attaching eBPF + to various hooks +* [perf](https://pkg.go.dev/github.com/cilium/ebpf/perf) allows reading from a + `PERF_EVENT_ARRAY` +* [cmd/bpf2go](https://pkg.go.dev/github.com/cilium/ebpf/cmd/bpf2go) allows + embedding eBPF in Go + +The library is maintained by [Cloudflare](https://www.cloudflare.com) and +[Cilium](https://www.cilium.io). Feel free to +[join](https://cilium.herokuapp.com/) the +[#libbpf-go](https://cilium.slack.com/messages/libbpf-go) channel on Slack. + +## Current status + +The package is production ready, but **the API is explicitly unstable right +now**. Expect to update your code if you want to follow along. + +## Requirements + +* A version of Go that is [supported by + upstream](https://golang.org/doc/devel/release.html#policy) +* Linux 4.9, 4.19 or 5.4 (versions in-between should work, but are not tested) + +## Useful resources + +* [eBPF.io](https://ebpf.io) (recommended) +* [Cilium eBPF documentation](https://docs.cilium.io/en/latest/bpf/#bpf-guide) + (recommended) +* [Linux documentation on + BPF](https://www.kernel.org/doc/html/latest/networking/filter.html) +* [eBPF features by Linux + version](https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md) + +## Regenerating Testdata + +Run `make` in the root of this repository to rebuild testdata in all +subpackages. This requires Docker, as it relies on a standardized build +environment to keep the build output stable. + +The toolchain image build files are kept in [testdata/docker/](testdata/docker/). + +## License + +MIT diff --git a/vendor/github.com/cilium/ebpf/abi.go b/vendor/github.com/cilium/ebpf/abi.go deleted file mode 100644 index d432eb46f420f..0000000000000 --- a/vendor/github.com/cilium/ebpf/abi.go +++ /dev/null @@ -1,206 +0,0 @@ -package ebpf - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "os" - "syscall" - - "github.com/cilium/ebpf/internal" -) - -// MapABI are the attributes of a Map which are available across all supported kernels. -type MapABI struct { - Type MapType - KeySize uint32 - ValueSize uint32 - MaxEntries uint32 - Flags uint32 -} - -func newMapABIFromSpec(spec *MapSpec) *MapABI { - return &MapABI{ - spec.Type, - spec.KeySize, - spec.ValueSize, - spec.MaxEntries, - spec.Flags, - } -} - -func newMapABIFromFd(fd *internal.FD) (string, *MapABI, error) { - info, err := bpfGetMapInfoByFD(fd) - if err != nil { - if errors.Is(err, syscall.EINVAL) { - abi, err := newMapABIFromProc(fd) - return "", abi, err - } - return "", nil, err - } - - return "", &MapABI{ - MapType(info.mapType), - info.keySize, - info.valueSize, - info.maxEntries, - info.flags, - }, nil -} - -func newMapABIFromProc(fd *internal.FD) (*MapABI, error) { - var abi MapABI - err := scanFdInfo(fd, map[string]interface{}{ - "map_type": &abi.Type, - "key_size": &abi.KeySize, - "value_size": &abi.ValueSize, - "max_entries": &abi.MaxEntries, - "map_flags": &abi.Flags, - }) - if err != nil { - return nil, err - } - return &abi, nil -} - -// Equal returns true if two ABIs have the same values. -func (abi *MapABI) Equal(other *MapABI) bool { - switch { - case abi.Type != other.Type: - return false - case abi.KeySize != other.KeySize: - return false - case abi.ValueSize != other.ValueSize: - return false - case abi.MaxEntries != other.MaxEntries: - return false - case abi.Flags != other.Flags: - return false - default: - return true - } -} - -// ProgramABI are the attributes of a Program which are available across all supported kernels. -type ProgramABI struct { - Type ProgramType -} - -func newProgramABIFromSpec(spec *ProgramSpec) *ProgramABI { - return &ProgramABI{ - spec.Type, - } -} - -func newProgramABIFromFd(fd *internal.FD) (string, *ProgramABI, error) { - info, err := bpfGetProgInfoByFD(fd) - if err != nil { - if errors.Is(err, syscall.EINVAL) { - return newProgramABIFromProc(fd) - } - - return "", nil, err - } - - var name string - if bpfName := internal.CString(info.name[:]); bpfName != "" { - name = bpfName - } else { - name = internal.CString(info.tag[:]) - } - - return name, &ProgramABI{ - Type: ProgramType(info.progType), - }, nil -} - -func newProgramABIFromProc(fd *internal.FD) (string, *ProgramABI, error) { - var ( - abi ProgramABI - name string - ) - - err := scanFdInfo(fd, map[string]interface{}{ - "prog_type": &abi.Type, - "prog_tag": &name, - }) - if errors.Is(err, errMissingFields) { - return "", nil, &internal.UnsupportedFeatureError{ - Name: "reading ABI from /proc/self/fdinfo", - MinimumVersion: internal.Version{4, 11, 0}, - } - } - if err != nil { - return "", nil, err - } - - return name, &abi, nil -} - -func scanFdInfo(fd *internal.FD, fields map[string]interface{}) error { - raw, err := fd.Value() - if err != nil { - return err - } - - fh, err := os.Open(fmt.Sprintf("/proc/self/fdinfo/%d", raw)) - if err != nil { - return err - } - defer fh.Close() - - if err := scanFdInfoReader(fh, fields); err != nil { - return fmt.Errorf("%s: %w", fh.Name(), err) - } - return nil -} - -var errMissingFields = errors.New("missing fields") - -func scanFdInfoReader(r io.Reader, fields map[string]interface{}) error { - var ( - scanner = bufio.NewScanner(r) - scanned int - ) - - for scanner.Scan() { - parts := bytes.SplitN(scanner.Bytes(), []byte("\t"), 2) - if len(parts) != 2 { - continue - } - - name := bytes.TrimSuffix(parts[0], []byte(":")) - field, ok := fields[string(name)] - if !ok { - continue - } - - if n, err := fmt.Fscanln(bytes.NewReader(parts[1]), field); err != nil || n != 1 { - return fmt.Errorf("can't parse field %s: %v", name, err) - } - - scanned++ - } - - if err := scanner.Err(); err != nil { - return err - } - - if scanned != len(fields) { - return errMissingFields - } - - return nil -} - -// Equal returns true if two ABIs have the same values. -func (abi *ProgramABI) Equal(other *ProgramABI) bool { - switch { - case abi.Type != other.Type: - return false - default: - return true - } -} diff --git a/vendor/github.com/cilium/ebpf/asm/instruction.go b/vendor/github.com/cilium/ebpf/asm/instruction.go index 8fbcf56647c7b..5d9d820e54f52 100644 --- a/vendor/github.com/cilium/ebpf/asm/instruction.go +++ b/vendor/github.com/cilium/ebpf/asm/instruction.go @@ -1,17 +1,29 @@ package asm import ( + "crypto/sha1" "encoding/binary" + "encoding/hex" "errors" "fmt" "io" "math" "strings" + + "github.com/cilium/ebpf/internal/unix" ) // InstructionSize is the size of a BPF instruction in bytes const InstructionSize = 8 +// RawInstructionOffset is an offset in units of raw BPF instructions. +type RawInstructionOffset uint64 + +// Bytes returns the offset of an instruction in bytes. +func (rio RawInstructionOffset) Bytes() uint64 { + return uint64(rio) * InstructionSize +} + // Instruction is a single eBPF instruction. type Instruction struct { OpCode OpCode @@ -151,10 +163,20 @@ func (ins *Instruction) mapOffset() uint32 { return uint32(uint64(ins.Constant) >> 32) } +// isLoadFromMap returns true if the instruction loads from a map. +// +// This covers both loading the map pointer and direct map value loads. func (ins *Instruction) isLoadFromMap() bool { return ins.OpCode == LoadImmOp(DWord) && (ins.Src == PseudoMapFD || ins.Src == PseudoMapValue) } +// IsFunctionCall returns true if the instruction calls another BPF function. +// +// This is not the same thing as a BPF helper call. +func (ins *Instruction) IsFunctionCall() bool { + return ins.OpCode.JumpOp() == Call && ins.Src == PseudoCall +} + // Format implements fmt.Formatter. func (ins Instruction) Format(f fmt.State, c rune) { if c != 'v' { @@ -310,28 +332,6 @@ func (insns Instructions) ReferenceOffsets() map[string][]int { return offsets } -func (insns Instructions) marshalledOffsets() (map[string]int, error) { - symbols := make(map[string]int) - - marshalledPos := 0 - for _, ins := range insns { - currentPos := marshalledPos - marshalledPos += ins.OpCode.marshalledInstructions() - - if ins.Symbol == "" { - continue - } - - if _, ok := symbols[ins.Symbol]; ok { - return nil, fmt.Errorf("duplicate symbol %s", ins.Symbol) - } - - symbols[ins.Symbol] = currentPos - } - - return symbols, nil -} - // Format implements fmt.Formatter. // // You can control indentation of symbols by @@ -370,21 +370,17 @@ func (insns Instructions) Format(f fmt.State, c rune) { symIndent = strings.Repeat(" ", symPadding) } - // Figure out how many digits we need to represent the highest - // offset. - highestOffset := 0 - for _, ins := range insns { - highestOffset += ins.OpCode.marshalledInstructions() - } + // Guess how many digits we need at most, by assuming that all instructions + // are double wide. + highestOffset := len(insns) * 2 offsetWidth := int(math.Ceil(math.Log10(float64(highestOffset)))) - offset := 0 - for _, ins := range insns { - if ins.Symbol != "" { - fmt.Fprintf(f, "%s%s:\n", symIndent, ins.Symbol) + iter := insns.Iterate() + for iter.Next() { + if iter.Ins.Symbol != "" { + fmt.Fprintf(f, "%s%s:\n", symIndent, iter.Ins.Symbol) } - fmt.Fprintf(f, "%s%*d: %v\n", indent, offsetWidth, offset, ins) - offset += ins.OpCode.marshalledInstructions() + fmt.Fprintf(f, "%s%*d: %v\n", indent, offsetWidth, iter.Offset, iter.Ins) } return @@ -392,41 +388,67 @@ func (insns Instructions) Format(f fmt.State, c rune) { // Marshal encodes a BPF program into the kernel format. func (insns Instructions) Marshal(w io.Writer, bo binary.ByteOrder) error { - absoluteOffsets, err := insns.marshalledOffsets() - if err != nil { - return err + for i, ins := range insns { + _, err := ins.Marshal(w, bo) + if err != nil { + return fmt.Errorf("instruction %d: %w", i, err) + } } + return nil +} - num := 0 +// Tag calculates the kernel tag for a series of instructions. +// +// It mirrors bpf_prog_calc_tag in the kernel and so can be compared +// to ProgramInfo.Tag to figure out whether a loaded program matches +// certain instructions. +func (insns Instructions) Tag(bo binary.ByteOrder) (string, error) { + h := sha1.New() for i, ins := range insns { - switch { - case ins.OpCode.JumpOp() == Call && ins.Src == PseudoCall && ins.Constant == -1: - // Rewrite bpf to bpf call - offset, ok := absoluteOffsets[ins.Reference] - if !ok { - return fmt.Errorf("instruction %d: reference to missing symbol %s", i, ins.Reference) - } - - ins.Constant = int64(offset - num - 1) - - case ins.OpCode.Class() == JumpClass && ins.Offset == -1: - // Rewrite jump to label - offset, ok := absoluteOffsets[ins.Reference] - if !ok { - return fmt.Errorf("instruction %d: reference to missing symbol %s", i, ins.Reference) - } - - ins.Offset = int16(offset - num - 1) + if ins.isLoadFromMap() { + ins.Constant = 0 } - - n, err := ins.Marshal(w, bo) + _, err := ins.Marshal(h, bo) if err != nil { - return fmt.Errorf("instruction %d: %w", i, err) + return "", fmt.Errorf("instruction %d: %w", i, err) } + } + return hex.EncodeToString(h.Sum(nil)[:unix.BPF_TAG_SIZE]), nil +} + +// Iterate allows iterating a BPF program while keeping track of +// various offsets. +// +// Modifying the instruction slice will lead to undefined behaviour. +func (insns Instructions) Iterate() *InstructionIterator { + return &InstructionIterator{insns: insns} +} - num += int(n / InstructionSize) +// InstructionIterator iterates over a BPF program. +type InstructionIterator struct { + insns Instructions + // The instruction in question. + Ins *Instruction + // The index of the instruction in the original instruction slice. + Index int + // The offset of the instruction in raw BPF instructions. This accounts + // for double-wide instructions. + Offset RawInstructionOffset +} + +// Next returns true as long as there are any instructions remaining. +func (iter *InstructionIterator) Next() bool { + if len(iter.insns) == 0 { + return false } - return nil + + if iter.Ins != nil { + iter.Index++ + iter.Offset += RawInstructionOffset(iter.Ins.OpCode.rawInstructions()) + } + iter.Ins = &iter.insns[0] + iter.insns = iter.insns[1:] + return true } type bpfInstruction struct { diff --git a/vendor/github.com/cilium/ebpf/asm/opcode.go b/vendor/github.com/cilium/ebpf/asm/opcode.go index c99b6595ac588..dc4564a98d20a 100644 --- a/vendor/github.com/cilium/ebpf/asm/opcode.go +++ b/vendor/github.com/cilium/ebpf/asm/opcode.go @@ -66,10 +66,10 @@ type OpCode uint8 // InvalidOpCode is returned by setters on OpCode const InvalidOpCode OpCode = 0xff -// marshalledInstructions returns the number of BPF instructions required +// rawInstructions returns the number of BPF instructions required // to encode this opcode. -func (op OpCode) marshalledInstructions() int { - if op == LoadImmOp(DWord) { +func (op OpCode) rawInstructions() int { + if op.isDWordLoad() { return 2 } return 1 diff --git a/vendor/github.com/cilium/ebpf/collection.go b/vendor/github.com/cilium/ebpf/collection.go index 0c8b65d94de21..8e362900326f7 100644 --- a/vendor/github.com/cilium/ebpf/collection.go +++ b/vendor/github.com/cilium/ebpf/collection.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "math" + "reflect" + "strings" "github.com/cilium/ebpf/asm" "github.com/cilium/ebpf/internal" @@ -11,7 +13,10 @@ import ( ) // CollectionOptions control loading a collection into the kernel. +// +// Maps and Programs are passed to NewMapWithOptions and NewProgramsWithOptions. type CollectionOptions struct { + Maps MapOptions Programs ProgramOptions } @@ -126,6 +131,106 @@ func (cs *CollectionSpec) RewriteConstants(consts map[string]interface{}) error return nil } +// Assign the contents of a CollectionSpec to a struct. +// +// This function is a short-cut to manually checking the presence +// of maps and programs in a collection spec. Consider using bpf2go if this +// sounds useful. +// +// The argument to must be a pointer to a struct. A field of the +// struct is updated with values from Programs or Maps if it +// has an `ebpf` tag and its type is *ProgramSpec or *MapSpec. +// The tag gives the name of the program or map as found in +// the CollectionSpec. +// +// struct { +// Foo *ebpf.ProgramSpec `ebpf:"xdp_foo"` +// Bar *ebpf.MapSpec `ebpf:"bar_map"` +// Ignored int +// } +// +// Returns an error if any of the fields can't be found, or +// if the same map or program is assigned multiple times. +func (cs *CollectionSpec) Assign(to interface{}) error { + valueOf := func(typ reflect.Type, name string) (reflect.Value, error) { + switch typ { + case reflect.TypeOf((*ProgramSpec)(nil)): + p := cs.Programs[name] + if p == nil { + return reflect.Value{}, fmt.Errorf("missing program %q", name) + } + return reflect.ValueOf(p), nil + case reflect.TypeOf((*MapSpec)(nil)): + m := cs.Maps[name] + if m == nil { + return reflect.Value{}, fmt.Errorf("missing map %q", name) + } + return reflect.ValueOf(m), nil + default: + return reflect.Value{}, fmt.Errorf("unsupported type %s", typ) + } + } + + return assignValues(to, valueOf) +} + +// LoadAndAssign maps and programs into the kernel and assign them to a struct. +// +// This function is a short-cut to manually checking the presence +// of maps and programs in a collection spec. Consider using bpf2go if this +// sounds useful. +// +// The argument to must be a pointer to a struct. A field of the +// struct is updated with values from Programs or Maps if it +// has an `ebpf` tag and its type is *Program or *Map. +// The tag gives the name of the program or map as found in +// the CollectionSpec. +// +// struct { +// Foo *ebpf.Program `ebpf:"xdp_foo"` +// Bar *ebpf.Map `ebpf:"bar_map"` +// Ignored int +// } +// +// opts may be nil. +// +// Returns an error if any of the fields can't be found, or +// if the same map or program is assigned multiple times. +func (cs *CollectionSpec) LoadAndAssign(to interface{}, opts *CollectionOptions) error { + if opts == nil { + opts = &CollectionOptions{} + } + + loadMap, loadProgram, done, cleanup := lazyLoadCollection(cs, opts) + defer cleanup() + + valueOf := func(typ reflect.Type, name string) (reflect.Value, error) { + switch typ { + case reflect.TypeOf((*Program)(nil)): + p, err := loadProgram(name) + if err != nil { + return reflect.Value{}, err + } + return reflect.ValueOf(p), nil + case reflect.TypeOf((*Map)(nil)): + m, err := loadMap(name) + if err != nil { + return reflect.Value{}, err + } + return reflect.ValueOf(m), nil + default: + return reflect.Value{}, fmt.Errorf("unsupported type %s", typ) + } + } + + if err := assignValues(to, valueOf); err != nil { + return err + } + + done() + return nil +} + // Collection is a collection of Programs and Maps associated // with their symbols type Collection struct { @@ -134,28 +239,75 @@ type Collection struct { } // NewCollection creates a Collection from a specification. -// -// Only maps referenced by at least one of the programs are initialized. func NewCollection(spec *CollectionSpec) (*Collection, error) { return NewCollectionWithOptions(spec, CollectionOptions{}) } // NewCollectionWithOptions creates a Collection from a specification. -// -// Only maps referenced by at least one of the programs are initialized. -func NewCollectionWithOptions(spec *CollectionSpec, opts CollectionOptions) (coll *Collection, err error) { +func NewCollectionWithOptions(spec *CollectionSpec, opts CollectionOptions) (*Collection, error) { + loadMap, loadProgram, done, cleanup := lazyLoadCollection(spec, &opts) + defer cleanup() + + for mapName := range spec.Maps { + _, err := loadMap(mapName) + if err != nil { + return nil, err + } + } + + for progName := range spec.Programs { + _, err := loadProgram(progName) + if err != nil { + return nil, err + } + } + + maps, progs := done() + return &Collection{ + progs, + maps, + }, nil +} + +type btfHandleCache map[*btf.Spec]*btf.Handle + +func (btfs btfHandleCache) load(spec *btf.Spec) (*btf.Handle, error) { + if btfs[spec] != nil { + return btfs[spec], nil + } + + handle, err := btf.NewHandle(spec) + if err != nil { + return nil, err + } + + btfs[spec] = handle + return handle, nil +} + +func (btfs btfHandleCache) close() { + for _, handle := range btfs { + handle.Close() + } +} + +func lazyLoadCollection(coll *CollectionSpec, opts *CollectionOptions) ( + loadMap func(string) (*Map, error), + loadProgram func(string) (*Program, error), + done func() (map[string]*Map, map[string]*Program), + cleanup func(), +) { var ( - maps = make(map[string]*Map) - progs = make(map[string]*Program) - btfs = make(map[*btf.Spec]*btf.Handle) + maps = make(map[string]*Map) + progs = make(map[string]*Program) + btfs = make(btfHandleCache) + skipMapsAndProgs = false ) - defer func() { - for _, btf := range btfs { - btf.Close() - } + cleanup = func() { + btfs.close() - if err == nil { + if skipMapsAndProgs { return } @@ -166,40 +318,43 @@ func NewCollectionWithOptions(spec *CollectionSpec, opts CollectionOptions) (col for _, p := range progs { p.Close() } - }() + } - loadBTF := func(spec *btf.Spec) (*btf.Handle, error) { - if btfs[spec] != nil { - return btfs[spec], nil - } + done = func() (map[string]*Map, map[string]*Program) { + skipMapsAndProgs = true + return maps, progs + } - handle, err := btf.NewHandle(spec) - if err != nil { - return nil, err + loadMap = func(mapName string) (*Map, error) { + if m := maps[mapName]; m != nil { + return m, nil } - btfs[spec] = handle - return handle, nil - } - - for mapName, mapSpec := range spec.Maps { - var handle *btf.Handle - if mapSpec.BTF != nil { - handle, err = loadBTF(btf.MapSpec(mapSpec.BTF)) - if err != nil && !errors.Is(err, btf.ErrNotSupported) { - return nil, err - } + mapSpec := coll.Maps[mapName] + if mapSpec == nil { + return nil, fmt.Errorf("missing map %s", mapName) } - m, err := newMapWithBTF(mapSpec, handle) + m, err := newMapWithOptions(mapSpec, opts.Maps, btfs) if err != nil { return nil, fmt.Errorf("map %s: %w", mapName, err) } + maps[mapName] = m + return m, nil } - for progName, origProgSpec := range spec.Programs { - progSpec := origProgSpec.Copy() + loadProgram = func(progName string) (*Program, error) { + if prog := progs[progName]; prog != nil { + return prog, nil + } + + progSpec := coll.Programs[progName] + if progSpec == nil { + return nil, fmt.Errorf("unknown program %s", progName) + } + + progSpec = progSpec.Copy() // Rewrite any reference to a valid map. for i := range progSpec.Instructions { @@ -215,9 +370,9 @@ func NewCollectionWithOptions(spec *CollectionSpec, opts CollectionOptions) (col continue } - m := maps[ins.Reference] - if m == nil { - return nil, fmt.Errorf("program %s: missing map %s", progName, ins.Reference) + m, err := loadMap(ins.Reference) + if err != nil { + return nil, fmt.Errorf("program %s: %s", progName, err) } fd := m.FD() @@ -229,25 +384,16 @@ func NewCollectionWithOptions(spec *CollectionSpec, opts CollectionOptions) (col } } - var handle *btf.Handle - if progSpec.BTF != nil { - handle, err = loadBTF(btf.ProgramSpec(progSpec.BTF)) - if err != nil && !errors.Is(err, btf.ErrNotSupported) { - return nil, err - } - } - - prog, err := newProgramWithBTF(progSpec, handle, opts.Programs) + prog, err := newProgramWithOptions(progSpec, opts.Programs, btfs) if err != nil { return nil, fmt.Errorf("program %s: %w", progName, err) } + progs[progName] = prog + return prog, nil } - return &Collection{ - progs, - maps, - }, nil + return } // LoadCollection parses an object file and converts it to a collection. @@ -292,3 +438,152 @@ func (coll *Collection) DetachProgram(name string) *Program { delete(coll.Programs, name) return p } + +// Assign the contents of a collection to a struct. +// +// Deprecated: use CollectionSpec.Assign instead. It provides the same +// functionality but creates only the maps and programs requested. +func (coll *Collection) Assign(to interface{}) error { + assignedMaps := make(map[string]struct{}) + assignedPrograms := make(map[string]struct{}) + valueOf := func(typ reflect.Type, name string) (reflect.Value, error) { + switch typ { + case reflect.TypeOf((*Program)(nil)): + p := coll.Programs[name] + if p == nil { + return reflect.Value{}, fmt.Errorf("missing program %q", name) + } + assignedPrograms[name] = struct{}{} + return reflect.ValueOf(p), nil + case reflect.TypeOf((*Map)(nil)): + m := coll.Maps[name] + if m == nil { + return reflect.Value{}, fmt.Errorf("missing map %q", name) + } + assignedMaps[name] = struct{}{} + return reflect.ValueOf(m), nil + default: + return reflect.Value{}, fmt.Errorf("unsupported type %s", typ) + } + } + + if err := assignValues(to, valueOf); err != nil { + return err + } + + for name := range assignedPrograms { + coll.DetachProgram(name) + } + + for name := range assignedMaps { + coll.DetachMap(name) + } + + return nil +} + +func assignValues(to interface{}, valueOf func(reflect.Type, string) (reflect.Value, error)) error { + type structField struct { + reflect.StructField + value reflect.Value + } + + var ( + fields []structField + visitedTypes = make(map[reflect.Type]bool) + flattenStruct func(reflect.Value) error + ) + + flattenStruct = func(structVal reflect.Value) error { + structType := structVal.Type() + if structType.Kind() != reflect.Struct { + return fmt.Errorf("%s is not a struct", structType) + } + + if visitedTypes[structType] { + return fmt.Errorf("recursion on type %s", structType) + } + + for i := 0; i < structType.NumField(); i++ { + field := structField{structType.Field(i), structVal.Field(i)} + + name := field.Tag.Get("ebpf") + if name != "" { + fields = append(fields, field) + continue + } + + var err error + switch field.Type.Kind() { + case reflect.Ptr: + if field.Type.Elem().Kind() != reflect.Struct { + continue + } + + if field.value.IsNil() { + return fmt.Errorf("nil pointer to %s", structType) + } + + err = flattenStruct(field.value.Elem()) + + case reflect.Struct: + err = flattenStruct(field.value) + + default: + continue + } + + if err != nil { + return fmt.Errorf("field %s: %s", field.Name, err) + } + } + + return nil + } + + toValue := reflect.ValueOf(to) + if toValue.Type().Kind() != reflect.Ptr { + return fmt.Errorf("%T is not a pointer to struct", to) + } + + if toValue.IsNil() { + return fmt.Errorf("nil pointer to %T", to) + } + + if err := flattenStruct(toValue.Elem()); err != nil { + return err + } + + type elem struct { + // Either *Map or *Program + typ reflect.Type + name string + } + + assignedTo := make(map[elem]string) + for _, field := range fields { + name := field.Tag.Get("ebpf") + if strings.Contains(name, ",") { + return fmt.Errorf("field %s: ebpf tag contains a comma", field.Name) + } + + e := elem{field.Type, name} + if assignedField := assignedTo[e]; assignedField != "" { + return fmt.Errorf("field %s: %q was already assigned to %s", field.Name, name, assignedField) + } + + value, err := valueOf(field.Type, name) + if err != nil { + return fmt.Errorf("field %s: %w", field.Name, err) + } + + if !field.value.CanSet() { + return fmt.Errorf("field %s: can't set value", field.Name) + } + + field.value.Set(value) + assignedTo[e] = field.Name + } + + return nil +} diff --git a/vendor/github.com/cilium/ebpf/doc.go b/vendor/github.com/cilium/ebpf/doc.go index d96e6b1e6d934..f7f34da8f44c2 100644 --- a/vendor/github.com/cilium/ebpf/doc.go +++ b/vendor/github.com/cilium/ebpf/doc.go @@ -12,6 +12,5 @@ // eBPF code should be compiled ahead of time using clang, and shipped with // your application as any other resource. // -// This package doesn't include code required to attach eBPF to Linux -// subsystems, since this varies per subsystem. +// Use the link subpackage to attach a loaded program to a hook in the kernel. package ebpf diff --git a/vendor/github.com/cilium/ebpf/elf_reader.go b/vendor/github.com/cilium/ebpf/elf_reader.go index 77acaed8d962b..943142c49e633 100644 --- a/vendor/github.com/cilium/ebpf/elf_reader.go +++ b/vendor/github.com/cilium/ebpf/elf_reader.go @@ -1,6 +1,7 @@ package ebpf import ( + "bufio" "bytes" "debug/elf" "encoding/binary" @@ -17,12 +18,14 @@ import ( "github.com/cilium/ebpf/internal/unix" ) +// elfCode is a convenience to reduce the amount of arguments that have to +// be passed around explicitly. You should treat it's contents as immutable. type elfCode struct { - *elf.File - symbols []elf.Symbol - symbolsPerSection map[elf.SectionIndex]map[uint64]elf.Symbol - license string - version uint32 + *internal.SafeELFFile + sections map[elf.SectionIndex]*elfSection + license string + version uint32 + btf *btf.Spec } // LoadCollectionSpec parses an ELF file into a CollectionSpec. @@ -42,63 +45,52 @@ func LoadCollectionSpec(file string) (*CollectionSpec, error) { // LoadCollectionSpecFromReader parses an ELF file into a CollectionSpec. func LoadCollectionSpecFromReader(rd io.ReaderAt) (*CollectionSpec, error) { - f, err := elf.NewFile(rd) + f, err := internal.NewSafeELFFile(rd) if err != nil { return nil, err } defer f.Close() - symbols, err := f.Symbols() - if err != nil { - return nil, fmt.Errorf("load symbols: %v", err) - } - - ec := &elfCode{f, symbols, symbolsPerSection(symbols), "", 0} - var ( licenseSection *elf.Section versionSection *elf.Section - btfMaps = make(map[elf.SectionIndex]*elf.Section) - progSections = make(map[elf.SectionIndex]*elf.Section) + sections = make(map[elf.SectionIndex]*elfSection) relSections = make(map[elf.SectionIndex]*elf.Section) - mapSections = make(map[elf.SectionIndex]*elf.Section) - dataSections = make(map[elf.SectionIndex]*elf.Section) ) - for i, sec := range ec.Sections { + // This is the target of relocations generated by inline assembly. + sections[elf.SHN_UNDEF] = newElfSection(new(elf.Section), undefSection) + + // Collect all the sections we're interested in. This includes relocations + // which we parse later. + for i, sec := range f.Sections { + idx := elf.SectionIndex(i) + switch { case strings.HasPrefix(sec.Name, "license"): licenseSection = sec case strings.HasPrefix(sec.Name, "version"): versionSection = sec case strings.HasPrefix(sec.Name, "maps"): - mapSections[elf.SectionIndex(i)] = sec + sections[idx] = newElfSection(sec, mapSection) case sec.Name == ".maps": - btfMaps[elf.SectionIndex(i)] = sec - case sec.Name == ".bss" || sec.Name == ".rodata" || sec.Name == ".data": - dataSections[elf.SectionIndex(i)] = sec + sections[idx] = newElfSection(sec, btfMapSection) + case sec.Name == ".bss" || sec.Name == ".data" || strings.HasPrefix(sec.Name, ".rodata"): + sections[idx] = newElfSection(sec, dataSection) case sec.Type == elf.SHT_REL: - if int(sec.Info) >= len(ec.Sections) { - return nil, fmt.Errorf("found relocation section %v for missing section %v", i, sec.Info) - } - // Store relocations under the section index of the target - idx := elf.SectionIndex(sec.Info) - if relSections[idx] != nil { - return nil, fmt.Errorf("section %d has multiple relocation sections", sec.Info) - } - relSections[idx] = sec + relSections[elf.SectionIndex(sec.Info)] = sec case sec.Type == elf.SHT_PROGBITS && (sec.Flags&elf.SHF_EXECINSTR) != 0 && sec.Size > 0: - progSections[elf.SectionIndex(i)] = sec + sections[idx] = newElfSection(sec, programSection) } } - ec.license, err = loadLicense(licenseSection) + license, err := loadLicense(licenseSection) if err != nil { return nil, fmt.Errorf("load license: %w", err) } - ec.version, err = loadVersion(versionSection, ec.ByteOrder) + version, err := loadVersion(versionSection, f.ByteOrder) if err != nil { return nil, fmt.Errorf("load version: %w", err) } @@ -108,37 +100,90 @@ func LoadCollectionSpecFromReader(rd io.ReaderAt) (*CollectionSpec, error) { return nil, fmt.Errorf("load BTF: %w", err) } - relocations, referencedSections, err := ec.loadRelocations(relSections) + // Assign symbols to all the sections we're interested in. + symbols, err := f.Symbols() if err != nil { - return nil, fmt.Errorf("load relocations: %w", err) + return nil, fmt.Errorf("load symbols: %v", err) } - maps := make(map[string]*MapSpec) - if err := ec.loadMaps(maps, mapSections); err != nil { - return nil, fmt.Errorf("load maps: %w", err) - } + for _, symbol := range symbols { + idx := symbol.Section + symType := elf.ST_TYPE(symbol.Info) - if len(btfMaps) > 0 { - if err := ec.loadBTFMaps(maps, btfMaps, btfSpec); err != nil { - return nil, fmt.Errorf("load BTF maps: %w", err) + section := sections[idx] + if section == nil { + continue } + + // Older versions of LLVM don't tag symbols correctly, so keep + // all NOTYPE ones. + keep := symType == elf.STT_NOTYPE + switch section.kind { + case mapSection, btfMapSection, dataSection: + keep = keep || symType == elf.STT_OBJECT + case programSection: + keep = keep || symType == elf.STT_FUNC + } + if !keep || symbol.Name == "" { + continue + } + + section.symbols[symbol.Value] = symbol } - if len(dataSections) > 0 { - for idx := range dataSections { - if !referencedSections[idx] { - // Prune data sections which are not referenced by any - // instructions. - delete(dataSections, idx) - } + ec := &elfCode{ + SafeELFFile: f, + sections: sections, + license: license, + version: version, + btf: btfSpec, + } + + // Go through relocation sections, and parse the ones for sections we're + // interested in. Make sure that relocations point at valid sections. + for idx, relSection := range relSections { + section := sections[idx] + if section == nil { + continue } - if err := ec.loadDataSections(maps, dataSections, btfSpec); err != nil { - return nil, fmt.Errorf("load data sections: %w", err) + rels, err := ec.loadRelocations(relSection, symbols) + if err != nil { + return nil, fmt.Errorf("relocation for section %q: %w", section.Name, err) + } + + for _, rel := range rels { + target := sections[rel.Section] + if target == nil { + return nil, fmt.Errorf("section %q: reference to %q in section %s: %w", section.Name, rel.Name, rel.Section, ErrNotSupported) + } + + if target.Flags&elf.SHF_STRINGS > 0 { + return nil, fmt.Errorf("section %q: string %q is not stack allocated: %w", section.Name, rel.Name, ErrNotSupported) + } + + target.references++ } + + section.relocations = rels + } + + // Collect all the various ways to define maps. + maps := make(map[string]*MapSpec) + if err := ec.loadMaps(maps); err != nil { + return nil, fmt.Errorf("load maps: %w", err) } - progs, err := ec.loadPrograms(progSections, relocations, btfSpec) + if err := ec.loadBTFMaps(maps); err != nil { + return nil, fmt.Errorf("load BTF maps: %w", err) + } + + if err := ec.loadDataSections(maps); err != nil { + return nil, fmt.Errorf("load data sections: %w", err) + } + + // Finally, collect programs and link them. + progs, err := ec.loadPrograms() if err != nil { return nil, fmt.Errorf("load programs: %w", err) } @@ -170,26 +215,61 @@ func loadVersion(sec *elf.Section, bo binary.ByteOrder) (uint32, error) { return version, nil } -func (ec *elfCode) loadPrograms(progSections map[elf.SectionIndex]*elf.Section, relocations map[elf.SectionIndex]map[uint64]elf.Symbol, btfSpec *btf.Spec) (map[string]*ProgramSpec, error) { +type elfSectionKind int + +const ( + undefSection elfSectionKind = iota + mapSection + btfMapSection + programSection + dataSection +) + +type elfSection struct { + *elf.Section + kind elfSectionKind + // Offset from the start of the section to a symbol + symbols map[uint64]elf.Symbol + // Offset from the start of the section to a relocation, which points at + // a symbol in another section. + relocations map[uint64]elf.Symbol + // The number of relocations pointing at this section. + references int +} + +func newElfSection(section *elf.Section, kind elfSectionKind) *elfSection { + return &elfSection{ + section, + kind, + make(map[uint64]elf.Symbol), + make(map[uint64]elf.Symbol), + 0, + } +} + +func (ec *elfCode) loadPrograms() (map[string]*ProgramSpec, error) { var ( progs []*ProgramSpec libs []*ProgramSpec ) - for idx, sec := range progSections { - syms := ec.symbolsPerSection[idx] - if len(syms) == 0 { + for _, sec := range ec.sections { + if sec.kind != programSection { + continue + } + + if len(sec.symbols) == 0 { return nil, fmt.Errorf("section %v: missing symbols", sec.Name) } - funcSym, ok := syms[0] + funcSym, ok := sec.symbols[0] if !ok { return nil, fmt.Errorf("section %v: no label at start", sec.Name) } - insns, length, err := ec.loadInstructions(sec, syms, relocations[idx]) + insns, length, err := ec.loadInstructions(sec) if err != nil { - return nil, fmt.Errorf("program %s: can't unmarshal instructions: %w", funcSym.Name, err) + return nil, fmt.Errorf("program %s: %w", funcSym.Name, err) } progType, attachType, attachTo := getProgType(sec.Name) @@ -205,8 +285,8 @@ func (ec *elfCode) loadPrograms(progSections map[elf.SectionIndex]*elf.Section, ByteOrder: ec.ByteOrder, } - if btfSpec != nil { - spec.BTF, err = btfSpec.Program(sec.Name, length) + if ec.btf != nil { + spec.BTF, err = ec.btf.Program(sec.Name, length) if err != nil && !errors.Is(err, btf.ErrNoExtendedInfo) { return nil, fmt.Errorf("program %s: %w", funcSym.Name, err) } @@ -234,9 +314,9 @@ func (ec *elfCode) loadPrograms(progSections map[elf.SectionIndex]*elf.Section, return res, nil } -func (ec *elfCode) loadInstructions(section *elf.Section, symbols, relocations map[uint64]elf.Symbol) (asm.Instructions, uint64, error) { +func (ec *elfCode) loadInstructions(section *elfSection) (asm.Instructions, uint64, error) { var ( - r = section.Open() + r = bufio.NewReader(section.Open()) insns asm.Instructions offset uint64 ) @@ -250,11 +330,11 @@ func (ec *elfCode) loadInstructions(section *elf.Section, symbols, relocations m return nil, 0, fmt.Errorf("offset %d: %w", offset, err) } - ins.Symbol = symbols[offset].Name + ins.Symbol = section.symbols[offset].Name - if rel, ok := relocations[offset]; ok { + if rel, ok := section.relocations[offset]; ok { if err = ec.relocateInstruction(&ins, rel); err != nil { - return nil, 0, fmt.Errorf("offset %d: can't relocate instruction: %w", offset, err) + return nil, 0, fmt.Errorf("offset %d: relocate instruction: %w", offset, err) } } @@ -270,69 +350,66 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err name = rel.Name ) - if typ == elf.STT_SECTION { - // Symbols with section type do not have a name set. Get it - // from the section itself. - idx := int(rel.Section) - if idx > len(ec.Sections) { - return errors.New("out-of-bounds section index") + target := ec.sections[rel.Section] + + switch target.kind { + case mapSection, btfMapSection: + if bind != elf.STB_GLOBAL { + return fmt.Errorf("possible erroneous static qualifier on map definition: found reference to %q", name) } - name = ec.Sections[idx].Name - } + if typ != elf.STT_OBJECT && typ != elf.STT_NOTYPE { + // STT_NOTYPE is generated on clang < 8 which doesn't tag + // relocations appropriately. + return fmt.Errorf("map load: incorrect relocation type %v", typ) + } + + ins.Src = asm.PseudoMapFD + + // Mark the instruction as needing an update when creating the + // collection. + if err := ins.RewriteMapPtr(-1); err != nil { + return err + } -outer: - switch { - case ins.OpCode == asm.LoadImmOp(asm.DWord): - // There are two distinct types of a load from a map: - // a direct one, where the value is extracted without - // a call to map_lookup_elem in eBPF, and an indirect one - // that goes via the helper. They are distinguished by - // different relocations. + case dataSection: switch typ { case elf.STT_SECTION: - // This is a direct load since the referenced symbol is a - // section. Weirdly, the offset of the real symbol in the - // section is encoded in the instruction stream. if bind != elf.STB_LOCAL { return fmt.Errorf("direct load: %s: unsupported relocation %s", name, bind) } - // For some reason, clang encodes the offset of the symbol its - // section in the first basic BPF instruction, while the kernel - // expects it in the second one. - ins.Constant <<= 32 - ins.Src = asm.PseudoMapValue - - case elf.STT_NOTYPE: - if bind == elf.STB_GLOBAL && rel.Section == elf.SHN_UNDEF { - // This is a relocation generated by inline assembly. - // We can't do more than assigning ins.Reference. - break outer - } - - // This is an ELF generated on clang < 8, which doesn't tag - // relocations appropriately. - fallthrough - case elf.STT_OBJECT: if bind != elf.STB_GLOBAL { - return fmt.Errorf("load: %s: unsupported binding: %s", name, bind) + return fmt.Errorf("direct load: %s: unsupported relocation %s", name, bind) } - ins.Src = asm.PseudoMapFD - default: - return fmt.Errorf("load: %s: unsupported relocation: %s", name, typ) + return fmt.Errorf("incorrect relocation type %v for direct map load", typ) } + // We rely on using the name of the data section as the reference. It + // would be nicer to keep the real name in case of an STT_OBJECT, but + // it's not clear how to encode that into Instruction. + name = target.Name + + // For some reason, clang encodes the offset of the symbol its + // section in the first basic BPF instruction, while the kernel + // expects it in the second one. + ins.Constant <<= 32 + ins.Src = asm.PseudoMapValue + // Mark the instruction as needing an update when creating the // collection. if err := ins.RewriteMapPtr(-1); err != nil { return err } - case ins.OpCode.JumpOp() == asm.Call: + case programSection: + if ins.OpCode.JumpOp() != asm.Call { + return fmt.Errorf("not a call instruction: %s", ins) + } + if ins.Src != asm.PseudoCall { return fmt.Errorf("call: %s: incorrect source register", name) } @@ -357,7 +434,7 @@ outer: return fmt.Errorf("call: %s: invalid offset %d", name, offset) } - sym, ok := ec.symbolsPerSection[rel.Section][uint64(offset)] + sym, ok := target.symbols[uint64(offset)] if !ok { return fmt.Errorf("call: %s: no symbol at offset %d", name, offset) } @@ -369,31 +446,46 @@ outer: return fmt.Errorf("call: %s: invalid symbol type %s", name, typ) } + case undefSection: + if bind != elf.STB_GLOBAL { + return fmt.Errorf("asm relocation: %s: unsupported binding: %s", name, bind) + } + + if typ != elf.STT_NOTYPE { + return fmt.Errorf("asm relocation: %s: unsupported type %s", name, typ) + } + + // There is nothing to do here but set ins.Reference. + default: - return fmt.Errorf("relocation for unsupported instruction: %s", ins.OpCode) + return fmt.Errorf("relocation to %q: %w", target.Name, ErrNotSupported) } ins.Reference = name return nil } -func (ec *elfCode) loadMaps(maps map[string]*MapSpec, mapSections map[elf.SectionIndex]*elf.Section) error { - for idx, sec := range mapSections { - syms := ec.symbolsPerSection[idx] - if len(syms) == 0 { +func (ec *elfCode) loadMaps(maps map[string]*MapSpec) error { + for _, sec := range ec.sections { + if sec.kind != mapSection { + continue + } + + nSym := len(sec.symbols) + if nSym == 0 { return fmt.Errorf("section %v: no symbols", sec.Name) } - if sec.Size%uint64(len(syms)) != 0 { + if sec.Size%uint64(nSym) != 0 { return fmt.Errorf("section %v: map descriptors are not of equal size", sec.Name) } var ( - r = sec.Open() - size = sec.Size / uint64(len(syms)) + r = bufio.NewReader(sec.Open()) + size = sec.Size / uint64(nSym) ) - for i, offset := 0, uint64(0); i < len(syms); i, offset = i+1, offset+size { - mapSym, ok := syms[offset] + for i, offset := 0, uint64(0); i < nSym; i, offset = i+1, offset+size { + mapSym, ok := sec.symbols[offset] if !ok { return fmt.Errorf("section %s: missing symbol for map at offset %d", sec.Name, offset) } @@ -431,24 +523,46 @@ func (ec *elfCode) loadMaps(maps map[string]*MapSpec, mapSections map[elf.Sectio return nil } -func (ec *elfCode) loadBTFMaps(maps map[string]*MapSpec, mapSections map[elf.SectionIndex]*elf.Section, spec *btf.Spec) error { - if spec == nil { - return fmt.Errorf("missing BTF") - } +func (ec *elfCode) loadBTFMaps(maps map[string]*MapSpec) error { + for _, sec := range ec.sections { + if sec.kind != btfMapSection { + continue + } + + if ec.btf == nil { + return fmt.Errorf("missing BTF") + } - for idx, sec := range mapSections { - syms := ec.symbolsPerSection[idx] - if len(syms) == 0 { + if len(sec.symbols) == 0 { return fmt.Errorf("section %v: no symbols", sec.Name) } - for _, sym := range syms { + _, err := io.Copy(internal.DiscardZeroes{}, bufio.NewReader(sec.Open())) + if err != nil { + return fmt.Errorf("section %v: initializing BTF map definitions: %w", sec.Name, internal.ErrNotSupported) + } + + for _, sym := range sec.symbols { name := sym.Name if maps[name] != nil { return fmt.Errorf("section %v: map %v already exists", sec.Name, sym) } - mapSpec, err := mapSpecFromBTF(spec, name) + // A global Var is created by declaring a struct with a 'structure variable', + // as is common in eBPF C to declare eBPF maps. For example, + // `struct { ... } map_name ...;` emits a global variable `map_name` + // with the type of said struct (which can be anonymous). + var v btf.Var + if err := ec.btf.FindType(name, &v); err != nil { + return fmt.Errorf("cannot find global variable '%s' in BTF: %w", name, err) + } + + mapStruct, ok := v.Type.(*btf.Struct) + if !ok { + return fmt.Errorf("expected struct, got %s", v.Type) + } + + mapSpec, err := mapSpecFromBTF(name, mapStruct, false, ec.btf) if err != nil { return fmt.Errorf("map %v: %w", name, err) } @@ -460,30 +574,21 @@ func (ec *elfCode) loadBTFMaps(maps map[string]*MapSpec, mapSections map[elf.Sec return nil } -func mapSpecFromBTF(spec *btf.Spec, name string) (*MapSpec, error) { - btfMap, btfMapMembers, err := spec.Map(name) - if err != nil { - return nil, fmt.Errorf("can't get BTF: %w", err) - } - - keyType := btf.MapKey(btfMap) - size, err := btf.Sizeof(keyType) - if err != nil { - return nil, fmt.Errorf("can't get size of BTF key: %w", err) - } - keySize := uint32(size) - - valueType := btf.MapValue(btfMap) - size, err = btf.Sizeof(valueType) - if err != nil { - return nil, fmt.Errorf("can't get size of BTF value: %w", err) - } - valueSize := uint32(size) +// mapSpecFromBTF produces a MapSpec based on a btf.Struct def representing +// a BTF map definition. The name and spec arguments will be copied to the +// resulting MapSpec, and inner must be true on any resursive invocations. +func mapSpecFromBTF(name string, def *btf.Struct, inner bool, spec *btf.Spec) (*MapSpec, error) { var ( + key, value btf.Type + keySize, valueSize uint32 mapType, flags, maxEntries uint32 + pinType PinType + innerMapSpec *MapSpec + err error ) - for _, member := range btfMapMembers { + + for i, member := range def.Members { switch member.Name { case "type": mapType, err = uintFromBTF(member.Type) @@ -503,8 +608,48 @@ func mapSpecFromBTF(spec *btf.Spec, name string) (*MapSpec, error) { return nil, fmt.Errorf("can't get BTF map max entries: %w", err) } + case "key": + if keySize != 0 { + return nil, errors.New("both key and key_size given") + } + + pk, ok := member.Type.(*btf.Pointer) + if !ok { + return nil, fmt.Errorf("key type is not a pointer: %T", member.Type) + } + + key = pk.Target + + size, err := btf.Sizeof(pk.Target) + if err != nil { + return nil, fmt.Errorf("can't get size of BTF key: %w", err) + } + + keySize = uint32(size) + + case "value": + if valueSize != 0 { + return nil, errors.New("both value and value_size given") + } + + vk, ok := member.Type.(*btf.Pointer) + if !ok { + return nil, fmt.Errorf("value type is not a pointer: %T", member.Type) + } + + value = vk.Target + + size, err := btf.Sizeof(vk.Target) + if err != nil { + return nil, fmt.Errorf("can't get size of BTF value: %w", err) + } + + valueSize = uint32(size) + case "key_size": - if _, isVoid := keyType.(*btf.Void); !isVoid { + // Key needs to be nil and keySize needs to be 0 for key_size to be + // considered a valid member. + if key != nil || keySize != 0 { return nil, errors.New("both key and key_size given") } @@ -514,7 +659,9 @@ func mapSpecFromBTF(spec *btf.Spec, name string) (*MapSpec, error) { } case "value_size": - if _, isVoid := valueType.(*btf.Void); !isVoid { + // Value needs to be nil and valueSize needs to be 0 for value_size to be + // considered a valid member. + if value != nil || valueSize != 0 { return nil, errors.New("both value and value_size given") } @@ -524,28 +671,79 @@ func mapSpecFromBTF(spec *btf.Spec, name string) (*MapSpec, error) { } case "pinning": + if inner { + return nil, errors.New("inner maps can't be pinned") + } + pinning, err := uintFromBTF(member.Type) if err != nil { return nil, fmt.Errorf("can't get pinning: %w", err) } - if pinning != 0 { - return nil, fmt.Errorf("'pinning' attribute not supported: %w", ErrNotSupported) + pinType = PinType(pinning) + + case "values": + // The 'values' field in BTF map definitions is used for declaring map + // value types that are references to other BPF objects, like other maps + // or programs. It is always expected to be an array of pointers. + if i != len(def.Members)-1 { + return nil, errors.New("'values' must be the last member in a BTF map definition") + } + + if valueSize != 0 && valueSize != 4 { + return nil, errors.New("value_size must be 0 or 4") + } + valueSize = 4 + + valueType, err := resolveBTFArrayMacro(member.Type) + if err != nil { + return nil, fmt.Errorf("can't resolve type of member 'values': %w", err) + } + + switch t := valueType.(type) { + case *btf.Struct: + // The values member pointing to an array of structs means we're expecting + // a map-in-map declaration. + if MapType(mapType) != ArrayOfMaps && MapType(mapType) != HashOfMaps { + return nil, errors.New("outer map needs to be an array or a hash of maps") + } + if inner { + return nil, fmt.Errorf("nested inner maps are not supported") + } + + // This inner map spec is used as a map template, but it needs to be + // created as a traditional map before it can be used to do so. + // libbpf names the inner map template '.inner', but we + // opted for _inner to simplify validation logic. (dots only supported + // on kernels 5.2 and up) + // Pass the BTF spec from the parent object, since both parent and + // child must be created from the same BTF blob (on kernels that support BTF). + innerMapSpec, err = mapSpecFromBTF(name+"_inner", t, true, spec) + if err != nil { + return nil, fmt.Errorf("can't parse BTF map definition of inner map: %w", err) + } + + default: + return nil, fmt.Errorf("unsupported value type %q in 'values' field", t) } - case "key", "value": default: return nil, fmt.Errorf("unrecognized field %s in BTF map definition", member.Name) } } + bm := btf.NewMap(spec, key, value) + return &MapSpec{ + Name: SanitizeName(name, -1), Type: MapType(mapType), KeySize: keySize, ValueSize: valueSize, MaxEntries: maxEntries, Flags: flags, - BTF: btfMap, + BTF: &bm, + Pinning: pinType, + InnerMap: innerMapSpec, }, nil } @@ -565,13 +763,40 @@ func uintFromBTF(typ btf.Type) (uint32, error) { return arr.Nelems, nil } -func (ec *elfCode) loadDataSections(maps map[string]*MapSpec, dataSections map[elf.SectionIndex]*elf.Section, spec *btf.Spec) error { - if spec == nil { - return errors.New("data sections require BTF, make sure all consts are marked as static") +// resolveBTFArrayMacro resolves the __array macro, which declares an array +// of pointers to a given type. This function returns the target Type of +// the pointers in the array. +func resolveBTFArrayMacro(typ btf.Type) (btf.Type, error) { + arr, ok := typ.(*btf.Array) + if !ok { + return nil, fmt.Errorf("not an array: %v", typ) } - for _, sec := range dataSections { - btfMap, err := spec.Datasec(sec.Name) + ptr, ok := arr.Type.(*btf.Pointer) + if !ok { + return nil, fmt.Errorf("not an array of pointers: %v", typ) + } + + return ptr.Target, nil +} + +func (ec *elfCode) loadDataSections(maps map[string]*MapSpec) error { + for _, sec := range ec.sections { + if sec.kind != dataSection { + continue + } + + if sec.references == 0 { + // Prune data sections which are not referenced by any + // instructions. + continue + } + + if ec.btf == nil { + return errors.New("data sections require BTF, make sure all consts are marked as static") + } + + btfMap, err := ec.btf.Datasec(sec.Name) if err != nil { return err } @@ -636,6 +861,8 @@ func getProgType(sectionName string) (ProgramType, AttachType, string) { "lirc_mode2": {LircMode2, AttachLircMode2}, "flow_dissector": {FlowDissector, AttachFlowDissector}, "iter/": {Tracing, AttachTraceIter}, + "sk_lookup/": {SkLookup, AttachSkLookup}, + "lsm/": {LSM, AttachLSMMac}, "cgroup_skb/ingress": {CGroupSKB, AttachCGroupInetIngress}, "cgroup_skb/egress": {CGroupSKB, AttachCGroupInetEgress}, @@ -674,69 +901,30 @@ func getProgType(sectionName string) (ProgramType, AttachType, string) { return UnspecifiedProgram, AttachNone, "" } -func (ec *elfCode) loadRelocations(sections map[elf.SectionIndex]*elf.Section) (map[elf.SectionIndex]map[uint64]elf.Symbol, map[elf.SectionIndex]bool, error) { - result := make(map[elf.SectionIndex]map[uint64]elf.Symbol) - targets := make(map[elf.SectionIndex]bool) - for idx, sec := range sections { - rels := make(map[uint64]elf.Symbol) - - if sec.Entsize < 16 { - return nil, nil, fmt.Errorf("section %s: relocations are less than 16 bytes", sec.Name) - } - - r := sec.Open() - for off := uint64(0); off < sec.Size; off += sec.Entsize { - ent := io.LimitReader(r, int64(sec.Entsize)) +func (ec *elfCode) loadRelocations(sec *elf.Section, symbols []elf.Symbol) (map[uint64]elf.Symbol, error) { + rels := make(map[uint64]elf.Symbol) - var rel elf.Rel64 - if binary.Read(ent, ec.ByteOrder, &rel) != nil { - return nil, nil, fmt.Errorf("can't parse relocation at offset %v", off) - } - - symNo := int(elf.R_SYM64(rel.Info) - 1) - if symNo >= len(ec.symbols) { - return nil, nil, fmt.Errorf("relocation at offset %d: symbol %v doesnt exist", off, symNo) - } - - symbol := ec.symbols[symNo] - targets[symbol.Section] = true - rels[rel.Off] = ec.symbols[symNo] - } - - result[idx] = rels + if sec.Entsize < 16 { + return nil, fmt.Errorf("section %s: relocations are less than 16 bytes", sec.Name) } - return result, targets, nil -} -func symbolsPerSection(symbols []elf.Symbol) map[elf.SectionIndex]map[uint64]elf.Symbol { - result := make(map[elf.SectionIndex]map[uint64]elf.Symbol) - for _, sym := range symbols { - switch elf.ST_TYPE(sym.Info) { - case elf.STT_NOTYPE: - // Older versions of LLVM doesn't tag - // symbols correctly. - break - case elf.STT_OBJECT: - break - case elf.STT_FUNC: - break - default: - continue - } + r := bufio.NewReader(sec.Open()) + for off := uint64(0); off < sec.Size; off += sec.Entsize { + ent := io.LimitReader(r, int64(sec.Entsize)) - if sym.Section == elf.SHN_UNDEF || sym.Section >= elf.SHN_LORESERVE { - continue + var rel elf.Rel64 + if binary.Read(ent, ec.ByteOrder, &rel) != nil { + return nil, fmt.Errorf("can't parse relocation at offset %v", off) } - if sym.Name == "" { - continue + symNo := int(elf.R_SYM64(rel.Info) - 1) + if symNo >= len(symbols) { + return nil, fmt.Errorf("offset %d: symbol %d doesn't exist", off, symNo) } - idx := sym.Section - if _, ok := result[idx]; !ok { - result[idx] = make(map[uint64]elf.Symbol) - } - result[idx][sym.Value] = sym + symbol := symbols[symNo] + rels[rel.Off] = symbol } - return result + + return rels, nil } diff --git a/vendor/github.com/cilium/ebpf/elf_reader_fuzz.go b/vendor/github.com/cilium/ebpf/elf_reader_fuzz.go new file mode 100644 index 0000000000000..d46d135f2fcd1 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/elf_reader_fuzz.go @@ -0,0 +1,21 @@ +// +build gofuzz + +// Use with https://github.com/dvyukov/go-fuzz + +package ebpf + +import "bytes" + +func FuzzLoadCollectionSpec(data []byte) int { + spec, err := LoadCollectionSpecFromReader(bytes.NewReader(data)) + if err != nil { + if spec != nil { + panic("spec is not nil") + } + return 0 + } + if spec == nil { + panic("spec is nil") + } + return 1 +} diff --git a/vendor/github.com/cilium/ebpf/examples/go.mod b/vendor/github.com/cilium/ebpf/examples/go.mod new file mode 100644 index 0000000000000..2e45866a82c99 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/examples/go.mod @@ -0,0 +1,9 @@ +module github.com/cilium/ebpf/examples + +go 1.15 + +require ( + github.com/cilium/ebpf v0.3.0 + github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 + golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 +) diff --git a/vendor/github.com/cilium/ebpf/examples/headers/bpf_helper_defs.h b/vendor/github.com/cilium/ebpf/examples/headers/bpf_helper_defs.h new file mode 100644 index 0000000000000..5ac02ece119eb --- /dev/null +++ b/vendor/github.com/cilium/ebpf/examples/headers/bpf_helper_defs.h @@ -0,0 +1,3265 @@ +/* This is auto-generated file. See bpf_helpers_doc.py for details. */ + +/* Forward declarations of BPF structs */ +struct bpf_fib_lookup; +struct bpf_sk_lookup; +struct bpf_perf_event_data; +struct bpf_perf_event_value; +struct bpf_pidns_info; +struct bpf_sock; +struct bpf_sock_addr; +struct bpf_sock_ops; +struct bpf_sock_tuple; +struct bpf_spin_lock; +struct bpf_sysctl; +struct bpf_tcp_sock; +struct bpf_tunnel_key; +struct bpf_xfrm_state; +struct pt_regs; +struct sk_reuseport_md; +struct sockaddr; +struct tcphdr; +struct seq_file; +struct tcp6_sock; +struct tcp_sock; +struct tcp_timewait_sock; +struct tcp_request_sock; +struct udp6_sock; +struct task_struct; +struct __sk_buff; +struct sk_msg_md; +struct xdp_md; + +/* + * bpf_map_lookup_elem + * + * Perform a lookup in *map* for an entry associated to *key*. + * + * Returns + * Map value associated to *key*, or **NULL** if no entry was + * found. + */ +static void *(*bpf_map_lookup_elem)(void *map, const void *key) = (void *) 1; + +/* + * bpf_map_update_elem + * + * Add or update the value of the entry associated to *key* in + * *map* with *value*. *flags* is one of: + * + * **BPF_NOEXIST** + * The entry for *key* must not exist in the map. + * **BPF_EXIST** + * The entry for *key* must already exist in the map. + * **BPF_ANY** + * No condition on the existence of the entry for *key*. + * + * Flag value **BPF_NOEXIST** cannot be used for maps of types + * **BPF_MAP_TYPE_ARRAY** or **BPF_MAP_TYPE_PERCPU_ARRAY** (all + * elements always exist), the helper would return an error. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_map_update_elem)(void *map, const void *key, const void *value, __u64 flags) = (void *) 2; + +/* + * bpf_map_delete_elem + * + * Delete entry with *key* from *map*. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_map_delete_elem)(void *map, const void *key) = (void *) 3; + +/* + * bpf_probe_read + * + * For tracing programs, safely attempt to read *size* bytes from + * kernel space address *unsafe_ptr* and store the data in *dst*. + * + * Generally, use **bpf_probe_read_user**\ () or + * **bpf_probe_read_kernel**\ () instead. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_probe_read)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 4; + +/* + * bpf_ktime_get_ns + * + * Return the time elapsed since system boot, in nanoseconds. + * Does not include time the system was suspended. + * See: **clock_gettime**\ (**CLOCK_MONOTONIC**) + * + * Returns + * Current *ktime*. + */ +static __u64 (*bpf_ktime_get_ns)(void) = (void *) 5; + +/* + * bpf_trace_printk + * + * This helper is a "printk()-like" facility for debugging. It + * prints a message defined by format *fmt* (of size *fmt_size*) + * to file *\/sys/kernel/debug/tracing/trace* from DebugFS, if + * available. It can take up to three additional **u64** + * arguments (as an eBPF helpers, the total number of arguments is + * limited to five). + * + * Each time the helper is called, it appends a line to the trace. + * Lines are discarded while *\/sys/kernel/debug/tracing/trace* is + * open, use *\/sys/kernel/debug/tracing/trace_pipe* to avoid this. + * The format of the trace is customizable, and the exact output + * one will get depends on the options set in + * *\/sys/kernel/debug/tracing/trace_options* (see also the + * *README* file under the same directory). However, it usually + * defaults to something like: + * + * :: + * + * telnet-470 [001] .N.. 419421.045894: 0x00000001: + * + * In the above: + * + * * ``telnet`` is the name of the current task. + * * ``470`` is the PID of the current task. + * * ``001`` is the CPU number on which the task is + * running. + * * In ``.N..``, each character refers to a set of + * options (whether irqs are enabled, scheduling + * options, whether hard/softirqs are running, level of + * preempt_disabled respectively). **N** means that + * **TIF_NEED_RESCHED** and **PREEMPT_NEED_RESCHED** + * are set. + * * ``419421.045894`` is a timestamp. + * * ``0x00000001`` is a fake value used by BPF for the + * instruction pointer register. + * * ```` is the message formatted with + * *fmt*. + * + * The conversion specifiers supported by *fmt* are similar, but + * more limited than for printk(). They are **%d**, **%i**, + * **%u**, **%x**, **%ld**, **%li**, **%lu**, **%lx**, **%lld**, + * **%lli**, **%llu**, **%llx**, **%p**, **%s**. No modifier (size + * of field, padding with zeroes, etc.) is available, and the + * helper will return **-EINVAL** (but print nothing) if it + * encounters an unknown specifier. + * + * Also, note that **bpf_trace_printk**\ () is slow, and should + * only be used for debugging purposes. For this reason, a notice + * bloc (spanning several lines) is printed to kernel logs and + * states that the helper should not be used "for production use" + * the first time this helper is used (or more precisely, when + * **trace_printk**\ () buffers are allocated). For passing values + * to user space, perf events should be preferred. + * + * Returns + * The number of bytes written to the buffer, or a negative error + * in case of failure. + */ +static long (*bpf_trace_printk)(const char *fmt, __u32 fmt_size, ...) = (void *) 6; + +/* + * bpf_get_prandom_u32 + * + * Get a pseudo-random number. + * + * From a security point of view, this helper uses its own + * pseudo-random internal state, and cannot be used to infer the + * seed of other random functions in the kernel. However, it is + * essential to note that the generator used by the helper is not + * cryptographically secure. + * + * Returns + * A random 32-bit unsigned value. + */ +static __u32 (*bpf_get_prandom_u32)(void) = (void *) 7; + +/* + * bpf_get_smp_processor_id + * + * Get the SMP (symmetric multiprocessing) processor id. Note that + * all programs run with preemption disabled, which means that the + * SMP processor id is stable during all the execution of the + * program. + * + * Returns + * The SMP id of the processor running the program. + */ +static __u32 (*bpf_get_smp_processor_id)(void) = (void *) 8; + +/* + * bpf_skb_store_bytes + * + * Store *len* bytes from address *from* into the packet + * associated to *skb*, at *offset*. *flags* are a combination of + * **BPF_F_RECOMPUTE_CSUM** (automatically recompute the + * checksum for the packet after storing the bytes) and + * **BPF_F_INVALIDATE_HASH** (set *skb*\ **->hash**, *skb*\ + * **->swhash** and *skb*\ **->l4hash** to 0). + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_store_bytes)(struct __sk_buff *skb, __u32 offset, const void *from, __u32 len, __u64 flags) = (void *) 9; + +/* + * bpf_l3_csum_replace + * + * Recompute the layer 3 (e.g. IP) checksum for the packet + * associated to *skb*. Computation is incremental, so the helper + * must know the former value of the header field that was + * modified (*from*), the new value of this field (*to*), and the + * number of bytes (2 or 4) for this field, stored in *size*. + * Alternatively, it is possible to store the difference between + * the previous and the new values of the header field in *to*, by + * setting *from* and *size* to 0. For both methods, *offset* + * indicates the location of the IP checksum within the packet. + * + * This helper works in combination with **bpf_csum_diff**\ (), + * which does not update the checksum in-place, but offers more + * flexibility and can handle sizes larger than 2 or 4 for the + * checksum to update. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_l3_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 size) = (void *) 10; + +/* + * bpf_l4_csum_replace + * + * Recompute the layer 4 (e.g. TCP, UDP or ICMP) checksum for the + * packet associated to *skb*. Computation is incremental, so the + * helper must know the former value of the header field that was + * modified (*from*), the new value of this field (*to*), and the + * number of bytes (2 or 4) for this field, stored on the lowest + * four bits of *flags*. Alternatively, it is possible to store + * the difference between the previous and the new values of the + * header field in *to*, by setting *from* and the four lowest + * bits of *flags* to 0. For both methods, *offset* indicates the + * location of the IP checksum within the packet. In addition to + * the size of the field, *flags* can be added (bitwise OR) actual + * flags. With **BPF_F_MARK_MANGLED_0**, a null checksum is left + * untouched (unless **BPF_F_MARK_ENFORCE** is added as well), and + * for updates resulting in a null checksum the value is set to + * **CSUM_MANGLED_0** instead. Flag **BPF_F_PSEUDO_HDR** indicates + * the checksum is to be computed against a pseudo-header. + * + * This helper works in combination with **bpf_csum_diff**\ (), + * which does not update the checksum in-place, but offers more + * flexibility and can handle sizes larger than 2 or 4 for the + * checksum to update. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_l4_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 flags) = (void *) 11; + +/* + * bpf_tail_call + * + * This special helper is used to trigger a "tail call", or in + * other words, to jump into another eBPF program. The same stack + * frame is used (but values on stack and in registers for the + * caller are not accessible to the callee). This mechanism allows + * for program chaining, either for raising the maximum number of + * available eBPF instructions, or to execute given programs in + * conditional blocks. For security reasons, there is an upper + * limit to the number of successive tail calls that can be + * performed. + * + * Upon call of this helper, the program attempts to jump into a + * program referenced at index *index* in *prog_array_map*, a + * special map of type **BPF_MAP_TYPE_PROG_ARRAY**, and passes + * *ctx*, a pointer to the context. + * + * If the call succeeds, the kernel immediately runs the first + * instruction of the new program. This is not a function call, + * and it never returns to the previous program. If the call + * fails, then the helper has no effect, and the caller continues + * to run its subsequent instructions. A call can fail if the + * destination program for the jump does not exist (i.e. *index* + * is superior to the number of entries in *prog_array_map*), or + * if the maximum number of tail calls has been reached for this + * chain of programs. This limit is defined in the kernel by the + * macro **MAX_TAIL_CALL_CNT** (not accessible to user space), + * which is currently set to 32. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_tail_call)(void *ctx, void *prog_array_map, __u32 index) = (void *) 12; + +/* + * bpf_clone_redirect + * + * Clone and redirect the packet associated to *skb* to another + * net device of index *ifindex*. Both ingress and egress + * interfaces can be used for redirection. The **BPF_F_INGRESS** + * value in *flags* is used to make the distinction (ingress path + * is selected if the flag is present, egress path otherwise). + * This is the only flag supported for now. + * + * In comparison with **bpf_redirect**\ () helper, + * **bpf_clone_redirect**\ () has the associated cost of + * duplicating the packet buffer, but this can be executed out of + * the eBPF program. Conversely, **bpf_redirect**\ () is more + * efficient, but it is handled through an action code where the + * redirection happens only after the eBPF program has returned. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_clone_redirect)(struct __sk_buff *skb, __u32 ifindex, __u64 flags) = (void *) 13; + +/* + * bpf_get_current_pid_tgid + * + * + * Returns + * A 64-bit integer containing the current tgid and pid, and + * created as such: + * *current_task*\ **->tgid << 32 \|** + * *current_task*\ **->pid**. + */ +static __u64 (*bpf_get_current_pid_tgid)(void) = (void *) 14; + +/* + * bpf_get_current_uid_gid + * + * + * Returns + * A 64-bit integer containing the current GID and UID, and + * created as such: *current_gid* **<< 32 \|** *current_uid*. + */ +static __u64 (*bpf_get_current_uid_gid)(void) = (void *) 15; + +/* + * bpf_get_current_comm + * + * Copy the **comm** attribute of the current task into *buf* of + * *size_of_buf*. The **comm** attribute contains the name of + * the executable (excluding the path) for the current task. The + * *size_of_buf* must be strictly positive. On success, the + * helper makes sure that the *buf* is NUL-terminated. On failure, + * it is filled with zeroes. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_get_current_comm)(void *buf, __u32 size_of_buf) = (void *) 16; + +/* + * bpf_get_cgroup_classid + * + * Retrieve the classid for the current task, i.e. for the net_cls + * cgroup to which *skb* belongs. + * + * This helper can be used on TC egress path, but not on ingress. + * + * The net_cls cgroup provides an interface to tag network packets + * based on a user-provided identifier for all traffic coming from + * the tasks belonging to the related cgroup. See also the related + * kernel documentation, available from the Linux sources in file + * *Documentation/admin-guide/cgroup-v1/net_cls.rst*. + * + * The Linux kernel has two versions for cgroups: there are + * cgroups v1 and cgroups v2. Both are available to users, who can + * use a mixture of them, but note that the net_cls cgroup is for + * cgroup v1 only. This makes it incompatible with BPF programs + * run on cgroups, which is a cgroup-v2-only feature (a socket can + * only hold data for one version of cgroups at a time). + * + * This helper is only available is the kernel was compiled with + * the **CONFIG_CGROUP_NET_CLASSID** configuration option set to + * "**y**" or to "**m**". + * + * Returns + * The classid, or 0 for the default unconfigured classid. + */ +static __u32 (*bpf_get_cgroup_classid)(struct __sk_buff *skb) = (void *) 17; + +/* + * bpf_skb_vlan_push + * + * Push a *vlan_tci* (VLAN tag control information) of protocol + * *vlan_proto* to the packet associated to *skb*, then update + * the checksum. Note that if *vlan_proto* is different from + * **ETH_P_8021Q** and **ETH_P_8021AD**, it is considered to + * be **ETH_P_8021Q**. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_vlan_push)(struct __sk_buff *skb, __be16 vlan_proto, __u16 vlan_tci) = (void *) 18; + +/* + * bpf_skb_vlan_pop + * + * Pop a VLAN header from the packet associated to *skb*. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_vlan_pop)(struct __sk_buff *skb) = (void *) 19; + +/* + * bpf_skb_get_tunnel_key + * + * Get tunnel metadata. This helper takes a pointer *key* to an + * empty **struct bpf_tunnel_key** of **size**, that will be + * filled with tunnel metadata for the packet associated to *skb*. + * The *flags* can be set to **BPF_F_TUNINFO_IPV6**, which + * indicates that the tunnel is based on IPv6 protocol instead of + * IPv4. + * + * The **struct bpf_tunnel_key** is an object that generalizes the + * principal parameters used by various tunneling protocols into a + * single struct. This way, it can be used to easily make a + * decision based on the contents of the encapsulation header, + * "summarized" in this struct. In particular, it holds the IP + * address of the remote end (IPv4 or IPv6, depending on the case) + * in *key*\ **->remote_ipv4** or *key*\ **->remote_ipv6**. Also, + * this struct exposes the *key*\ **->tunnel_id**, which is + * generally mapped to a VNI (Virtual Network Identifier), making + * it programmable together with the **bpf_skb_set_tunnel_key**\ + * () helper. + * + * Let's imagine that the following code is part of a program + * attached to the TC ingress interface, on one end of a GRE + * tunnel, and is supposed to filter out all messages coming from + * remote ends with IPv4 address other than 10.0.0.1: + * + * :: + * + * int ret; + * struct bpf_tunnel_key key = {}; + * + * ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key), 0); + * if (ret < 0) + * return TC_ACT_SHOT; // drop packet + * + * if (key.remote_ipv4 != 0x0a000001) + * return TC_ACT_SHOT; // drop packet + * + * return TC_ACT_OK; // accept packet + * + * This interface can also be used with all encapsulation devices + * that can operate in "collect metadata" mode: instead of having + * one network device per specific configuration, the "collect + * metadata" mode only requires a single device where the + * configuration can be extracted from this helper. + * + * This can be used together with various tunnels such as VXLan, + * Geneve, GRE or IP in IP (IPIP). + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_get_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_key *key, __u32 size, __u64 flags) = (void *) 20; + +/* + * bpf_skb_set_tunnel_key + * + * Populate tunnel metadata for packet associated to *skb.* The + * tunnel metadata is set to the contents of *key*, of *size*. The + * *flags* can be set to a combination of the following values: + * + * **BPF_F_TUNINFO_IPV6** + * Indicate that the tunnel is based on IPv6 protocol + * instead of IPv4. + * **BPF_F_ZERO_CSUM_TX** + * For IPv4 packets, add a flag to tunnel metadata + * indicating that checksum computation should be skipped + * and checksum set to zeroes. + * **BPF_F_DONT_FRAGMENT** + * Add a flag to tunnel metadata indicating that the + * packet should not be fragmented. + * **BPF_F_SEQ_NUMBER** + * Add a flag to tunnel metadata indicating that a + * sequence number should be added to tunnel header before + * sending the packet. This flag was added for GRE + * encapsulation, but might be used with other protocols + * as well in the future. + * + * Here is a typical usage on the transmit path: + * + * :: + * + * struct bpf_tunnel_key key; + * populate key ... + * bpf_skb_set_tunnel_key(skb, &key, sizeof(key), 0); + * bpf_clone_redirect(skb, vxlan_dev_ifindex, 0); + * + * See also the description of the **bpf_skb_get_tunnel_key**\ () + * helper for additional information. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_set_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_key *key, __u32 size, __u64 flags) = (void *) 21; + +/* + * bpf_perf_event_read + * + * Read the value of a perf event counter. This helper relies on a + * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. The nature of + * the perf event counter is selected when *map* is updated with + * perf event file descriptors. The *map* is an array whose size + * is the number of available CPUs, and each cell contains a value + * relative to one CPU. The value to retrieve is indicated by + * *flags*, that contains the index of the CPU to look up, masked + * with **BPF_F_INDEX_MASK**. Alternatively, *flags* can be set to + * **BPF_F_CURRENT_CPU** to indicate that the value for the + * current CPU should be retrieved. + * + * Note that before Linux 4.13, only hardware perf event can be + * retrieved. + * + * Also, be aware that the newer helper + * **bpf_perf_event_read_value**\ () is recommended over + * **bpf_perf_event_read**\ () in general. The latter has some ABI + * quirks where error and counter value are used as a return code + * (which is wrong to do since ranges may overlap). This issue is + * fixed with **bpf_perf_event_read_value**\ (), which at the same + * time provides more features over the **bpf_perf_event_read**\ + * () interface. Please refer to the description of + * **bpf_perf_event_read_value**\ () for details. + * + * Returns + * The value of the perf event counter read from the map, or a + * negative error code in case of failure. + */ +static __u64 (*bpf_perf_event_read)(void *map, __u64 flags) = (void *) 22; + +/* + * bpf_redirect + * + * Redirect the packet to another net device of index *ifindex*. + * This helper is somewhat similar to **bpf_clone_redirect**\ + * (), except that the packet is not cloned, which provides + * increased performance. + * + * Except for XDP, both ingress and egress interfaces can be used + * for redirection. The **BPF_F_INGRESS** value in *flags* is used + * to make the distinction (ingress path is selected if the flag + * is present, egress path otherwise). Currently, XDP only + * supports redirection to the egress interface, and accepts no + * flag at all. + * + * The same effect can also be attained with the more generic + * **bpf_redirect_map**\ (), which uses a BPF map to store the + * redirect target instead of providing it directly to the helper. + * + * Returns + * For XDP, the helper returns **XDP_REDIRECT** on success or + * **XDP_ABORTED** on error. For other program types, the values + * are **TC_ACT_REDIRECT** on success or **TC_ACT_SHOT** on + * error. + */ +static long (*bpf_redirect)(__u32 ifindex, __u64 flags) = (void *) 23; + +/* + * bpf_get_route_realm + * + * Retrieve the realm or the route, that is to say the + * **tclassid** field of the destination for the *skb*. The + * indentifier retrieved is a user-provided tag, similar to the + * one used with the net_cls cgroup (see description for + * **bpf_get_cgroup_classid**\ () helper), but here this tag is + * held by a route (a destination entry), not by a task. + * + * Retrieving this identifier works with the clsact TC egress hook + * (see also **tc-bpf(8)**), or alternatively on conventional + * classful egress qdiscs, but not on TC ingress path. In case of + * clsact TC egress hook, this has the advantage that, internally, + * the destination entry has not been dropped yet in the transmit + * path. Therefore, the destination entry does not need to be + * artificially held via **netif_keep_dst**\ () for a classful + * qdisc until the *skb* is freed. + * + * This helper is available only if the kernel was compiled with + * **CONFIG_IP_ROUTE_CLASSID** configuration option. + * + * Returns + * The realm of the route for the packet associated to *skb*, or 0 + * if none was found. + */ +static __u32 (*bpf_get_route_realm)(struct __sk_buff *skb) = (void *) 24; + +/* + * bpf_perf_event_output + * + * Write raw *data* blob into a special BPF perf event held by + * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf + * event must have the following attributes: **PERF_SAMPLE_RAW** + * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and + * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. + * + * The *flags* are used to indicate the index in *map* for which + * the value must be put, masked with **BPF_F_INDEX_MASK**. + * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** + * to indicate that the index of the current CPU core should be + * used. + * + * The value to write, of *size*, is passed through eBPF stack and + * pointed by *data*. + * + * The context of the program *ctx* needs also be passed to the + * helper. + * + * On user space, a program willing to read the values needs to + * call **perf_event_open**\ () on the perf event (either for + * one or for all CPUs) and to store the file descriptor into the + * *map*. This must be done before the eBPF program can send data + * into it. An example is available in file + * *samples/bpf/trace_output_user.c* in the Linux kernel source + * tree (the eBPF program counterpart is in + * *samples/bpf/trace_output_kern.c*). + * + * **bpf_perf_event_output**\ () achieves better performance + * than **bpf_trace_printk**\ () for sharing data with user + * space, and is much better suitable for streaming data from eBPF + * programs. + * + * Note that this helper is not restricted to tracing use cases + * and can be used with programs attached to TC or XDP as well, + * where it allows for passing data to user space listeners. Data + * can be: + * + * * Only custom structs, + * * Only the packet payload, or + * * A combination of both. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_perf_event_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 25; + +/* + * bpf_skb_load_bytes + * + * This helper was provided as an easy way to load data from a + * packet. It can be used to load *len* bytes from *offset* from + * the packet associated to *skb*, into the buffer pointed by + * *to*. + * + * Since Linux 4.7, usage of this helper has mostly been replaced + * by "direct packet access", enabling packet data to be + * manipulated with *skb*\ **->data** and *skb*\ **->data_end** + * pointing respectively to the first byte of packet data and to + * the byte after the last byte of packet data. However, it + * remains useful if one wishes to read large quantities of data + * at once from a packet into the eBPF stack. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_load_bytes)(const void *skb, __u32 offset, void *to, __u32 len) = (void *) 26; + +/* + * bpf_get_stackid + * + * Walk a user or a kernel stack and return its id. To achieve + * this, the helper needs *ctx*, which is a pointer to the context + * on which the tracing program is executed, and a pointer to a + * *map* of type **BPF_MAP_TYPE_STACK_TRACE**. + * + * The last argument, *flags*, holds the number of stack frames to + * skip (from 0 to 255), masked with + * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set + * a combination of the following flags: + * + * **BPF_F_USER_STACK** + * Collect a user space stack instead of a kernel stack. + * **BPF_F_FAST_STACK_CMP** + * Compare stacks by hash only. + * **BPF_F_REUSE_STACKID** + * If two different stacks hash into the same *stackid*, + * discard the old one. + * + * The stack id retrieved is a 32 bit long integer handle which + * can be further combined with other data (including other stack + * ids) and used as a key into maps. This can be useful for + * generating a variety of graphs (such as flame graphs or off-cpu + * graphs). + * + * For walking a stack, this helper is an improvement over + * **bpf_probe_read**\ (), which can be used with unrolled loops + * but is not efficient and consumes a lot of eBPF instructions. + * Instead, **bpf_get_stackid**\ () can collect up to + * **PERF_MAX_STACK_DEPTH** both kernel and user frames. Note that + * this limit can be controlled with the **sysctl** program, and + * that it should be manually increased in order to profile long + * user stacks (such as stacks for Java programs). To do so, use: + * + * :: + * + * # sysctl kernel.perf_event_max_stack= + * + * Returns + * The positive or null stack id on success, or a negative error + * in case of failure. + */ +static long (*bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void *) 27; + +/* + * bpf_csum_diff + * + * Compute a checksum difference, from the raw buffer pointed by + * *from*, of length *from_size* (that must be a multiple of 4), + * towards the raw buffer pointed by *to*, of size *to_size* + * (same remark). An optional *seed* can be added to the value + * (this can be cascaded, the seed may come from a previous call + * to the helper). + * + * This is flexible enough to be used in several ways: + * + * * With *from_size* == 0, *to_size* > 0 and *seed* set to + * checksum, it can be used when pushing new data. + * * With *from_size* > 0, *to_size* == 0 and *seed* set to + * checksum, it can be used when removing data from a packet. + * * With *from_size* > 0, *to_size* > 0 and *seed* set to 0, it + * can be used to compute a diff. Note that *from_size* and + * *to_size* do not need to be equal. + * + * This helper can be used in combination with + * **bpf_l3_csum_replace**\ () and **bpf_l4_csum_replace**\ (), to + * which one can feed in the difference computed with + * **bpf_csum_diff**\ (). + * + * Returns + * The checksum result, or a negative error code in case of + * failure. + */ +static __s64 (*bpf_csum_diff)(__be32 *from, __u32 from_size, __be32 *to, __u32 to_size, __wsum seed) = (void *) 28; + +/* + * bpf_skb_get_tunnel_opt + * + * Retrieve tunnel options metadata for the packet associated to + * *skb*, and store the raw tunnel option data to the buffer *opt* + * of *size*. + * + * This helper can be used with encapsulation devices that can + * operate in "collect metadata" mode (please refer to the related + * note in the description of **bpf_skb_get_tunnel_key**\ () for + * more details). A particular example where this can be used is + * in combination with the Geneve encapsulation protocol, where it + * allows for pushing (with **bpf_skb_get_tunnel_opt**\ () helper) + * and retrieving arbitrary TLVs (Type-Length-Value headers) from + * the eBPF program. This allows for full customization of these + * headers. + * + * Returns + * The size of the option data retrieved. + */ +static long (*bpf_skb_get_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 size) = (void *) 29; + +/* + * bpf_skb_set_tunnel_opt + * + * Set tunnel options metadata for the packet associated to *skb* + * to the option data contained in the raw buffer *opt* of *size*. + * + * See also the description of the **bpf_skb_get_tunnel_opt**\ () + * helper for additional information. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_set_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 size) = (void *) 30; + +/* + * bpf_skb_change_proto + * + * Change the protocol of the *skb* to *proto*. Currently + * supported are transition from IPv4 to IPv6, and from IPv6 to + * IPv4. The helper takes care of the groundwork for the + * transition, including resizing the socket buffer. The eBPF + * program is expected to fill the new headers, if any, via + * **skb_store_bytes**\ () and to recompute the checksums with + * **bpf_l3_csum_replace**\ () and **bpf_l4_csum_replace**\ + * (). The main case for this helper is to perform NAT64 + * operations out of an eBPF program. + * + * Internally, the GSO type is marked as dodgy so that headers are + * checked and segments are recalculated by the GSO/GRO engine. + * The size for GSO target is adapted as well. + * + * All values for *flags* are reserved for future usage, and must + * be left at zero. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_change_proto)(struct __sk_buff *skb, __be16 proto, __u64 flags) = (void *) 31; + +/* + * bpf_skb_change_type + * + * Change the packet type for the packet associated to *skb*. This + * comes down to setting *skb*\ **->pkt_type** to *type*, except + * the eBPF program does not have a write access to *skb*\ + * **->pkt_type** beside this helper. Using a helper here allows + * for graceful handling of errors. + * + * The major use case is to change incoming *skb*s to + * **PACKET_HOST** in a programmatic way instead of having to + * recirculate via **redirect**\ (..., **BPF_F_INGRESS**), for + * example. + * + * Note that *type* only allows certain values. At this time, they + * are: + * + * **PACKET_HOST** + * Packet is for us. + * **PACKET_BROADCAST** + * Send packet to all. + * **PACKET_MULTICAST** + * Send packet to group. + * **PACKET_OTHERHOST** + * Send packet to someone else. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_change_type)(struct __sk_buff *skb, __u32 type) = (void *) 32; + +/* + * bpf_skb_under_cgroup + * + * Check whether *skb* is a descendant of the cgroup2 held by + * *map* of type **BPF_MAP_TYPE_CGROUP_ARRAY**, at *index*. + * + * Returns + * The return value depends on the result of the test, and can be: + * + * * 0, if the *skb* failed the cgroup2 descendant test. + * * 1, if the *skb* succeeded the cgroup2 descendant test. + * * A negative error code, if an error occurred. + */ +static long (*bpf_skb_under_cgroup)(struct __sk_buff *skb, void *map, __u32 index) = (void *) 33; + +/* + * bpf_get_hash_recalc + * + * Retrieve the hash of the packet, *skb*\ **->hash**. If it is + * not set, in particular if the hash was cleared due to mangling, + * recompute this hash. Later accesses to the hash can be done + * directly with *skb*\ **->hash**. + * + * Calling **bpf_set_hash_invalid**\ (), changing a packet + * prototype with **bpf_skb_change_proto**\ (), or calling + * **bpf_skb_store_bytes**\ () with the + * **BPF_F_INVALIDATE_HASH** are actions susceptible to clear + * the hash and to trigger a new computation for the next call to + * **bpf_get_hash_recalc**\ (). + * + * Returns + * The 32-bit hash. + */ +static __u32 (*bpf_get_hash_recalc)(struct __sk_buff *skb) = (void *) 34; + +/* + * bpf_get_current_task + * + * + * Returns + * A pointer to the current task struct. + */ +static __u64 (*bpf_get_current_task)(void) = (void *) 35; + +/* + * bpf_probe_write_user + * + * Attempt in a safe way to write *len* bytes from the buffer + * *src* to *dst* in memory. It only works for threads that are in + * user context, and *dst* must be a valid user space address. + * + * This helper should not be used to implement any kind of + * security mechanism because of TOC-TOU attacks, but rather to + * debug, divert, and manipulate execution of semi-cooperative + * processes. + * + * Keep in mind that this feature is meant for experiments, and it + * has a risk of crashing the system and running programs. + * Therefore, when an eBPF program using this helper is attached, + * a warning including PID and process name is printed to kernel + * logs. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_probe_write_user)(void *dst, const void *src, __u32 len) = (void *) 36; + +/* + * bpf_current_task_under_cgroup + * + * Check whether the probe is being run is the context of a given + * subset of the cgroup2 hierarchy. The cgroup2 to test is held by + * *map* of type **BPF_MAP_TYPE_CGROUP_ARRAY**, at *index*. + * + * Returns + * The return value depends on the result of the test, and can be: + * + * * 0, if the *skb* task belongs to the cgroup2. + * * 1, if the *skb* task does not belong to the cgroup2. + * * A negative error code, if an error occurred. + */ +static long (*bpf_current_task_under_cgroup)(void *map, __u32 index) = (void *) 37; + +/* + * bpf_skb_change_tail + * + * Resize (trim or grow) the packet associated to *skb* to the + * new *len*. The *flags* are reserved for future usage, and must + * be left at zero. + * + * The basic idea is that the helper performs the needed work to + * change the size of the packet, then the eBPF program rewrites + * the rest via helpers like **bpf_skb_store_bytes**\ (), + * **bpf_l3_csum_replace**\ (), **bpf_l3_csum_replace**\ () + * and others. This helper is a slow path utility intended for + * replies with control messages. And because it is targeted for + * slow path, the helper itself can afford to be slow: it + * implicitly linearizes, unclones and drops offloads from the + * *skb*. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_change_tail)(struct __sk_buff *skb, __u32 len, __u64 flags) = (void *) 38; + +/* + * bpf_skb_pull_data + * + * Pull in non-linear data in case the *skb* is non-linear and not + * all of *len* are part of the linear section. Make *len* bytes + * from *skb* readable and writable. If a zero value is passed for + * *len*, then the whole length of the *skb* is pulled. + * + * This helper is only needed for reading and writing with direct + * packet access. + * + * For direct packet access, testing that offsets to access + * are within packet boundaries (test on *skb*\ **->data_end**) is + * susceptible to fail if offsets are invalid, or if the requested + * data is in non-linear parts of the *skb*. On failure the + * program can just bail out, or in the case of a non-linear + * buffer, use a helper to make the data available. The + * **bpf_skb_load_bytes**\ () helper is a first solution to access + * the data. Another one consists in using **bpf_skb_pull_data** + * to pull in once the non-linear parts, then retesting and + * eventually access the data. + * + * At the same time, this also makes sure the *skb* is uncloned, + * which is a necessary condition for direct write. As this needs + * to be an invariant for the write part only, the verifier + * detects writes and adds a prologue that is calling + * **bpf_skb_pull_data()** to effectively unclone the *skb* from + * the very beginning in case it is indeed cloned. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_pull_data)(struct __sk_buff *skb, __u32 len) = (void *) 39; + +/* + * bpf_csum_update + * + * Add the checksum *csum* into *skb*\ **->csum** in case the + * driver has supplied a checksum for the entire packet into that + * field. Return an error otherwise. This helper is intended to be + * used in combination with **bpf_csum_diff**\ (), in particular + * when the checksum needs to be updated after data has been + * written into the packet through direct packet access. + * + * Returns + * The checksum on success, or a negative error code in case of + * failure. + */ +static __s64 (*bpf_csum_update)(struct __sk_buff *skb, __wsum csum) = (void *) 40; + +/* + * bpf_set_hash_invalid + * + * Invalidate the current *skb*\ **->hash**. It can be used after + * mangling on headers through direct packet access, in order to + * indicate that the hash is outdated and to trigger a + * recalculation the next time the kernel tries to access this + * hash or when the **bpf_get_hash_recalc**\ () helper is called. + * + */ +static void (*bpf_set_hash_invalid)(struct __sk_buff *skb) = (void *) 41; + +/* + * bpf_get_numa_node_id + * + * Return the id of the current NUMA node. The primary use case + * for this helper is the selection of sockets for the local NUMA + * node, when the program is attached to sockets using the + * **SO_ATTACH_REUSEPORT_EBPF** option (see also **socket(7)**), + * but the helper is also available to other eBPF program types, + * similarly to **bpf_get_smp_processor_id**\ (). + * + * Returns + * The id of current NUMA node. + */ +static long (*bpf_get_numa_node_id)(void) = (void *) 42; + +/* + * bpf_skb_change_head + * + * Grows headroom of packet associated to *skb* and adjusts the + * offset of the MAC header accordingly, adding *len* bytes of + * space. It automatically extends and reallocates memory as + * required. + * + * This helper can be used on a layer 3 *skb* to push a MAC header + * for redirection into a layer 2 device. + * + * All values for *flags* are reserved for future usage, and must + * be left at zero. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_change_head)(struct __sk_buff *skb, __u32 len, __u64 flags) = (void *) 43; + +/* + * bpf_xdp_adjust_head + * + * Adjust (move) *xdp_md*\ **->data** by *delta* bytes. Note that + * it is possible to use a negative value for *delta*. This helper + * can be used to prepare the packet for pushing or popping + * headers. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_xdp_adjust_head)(struct xdp_md *xdp_md, int delta) = (void *) 44; + +/* + * bpf_probe_read_str + * + * Copy a NUL terminated string from an unsafe kernel address + * *unsafe_ptr* to *dst*. See **bpf_probe_read_kernel_str**\ () for + * more details. + * + * Generally, use **bpf_probe_read_user_str**\ () or + * **bpf_probe_read_kernel_str**\ () instead. + * + * Returns + * On success, the strictly positive length of the string, + * including the trailing NUL character. On error, a negative + * value. + */ +static long (*bpf_probe_read_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 45; + +/* + * bpf_get_socket_cookie + * + * If the **struct sk_buff** pointed by *skb* has a known socket, + * retrieve the cookie (generated by the kernel) of this socket. + * If no cookie has been set yet, generate a new cookie. Once + * generated, the socket cookie remains stable for the life of the + * socket. This helper can be useful for monitoring per socket + * networking traffic statistics as it provides a global socket + * identifier that can be assumed unique. + * + * Returns + * A 8-byte long non-decreasing number on success, or 0 if the + * socket field is missing inside *skb*. + */ +static __u64 (*bpf_get_socket_cookie)(void *ctx) = (void *) 46; + +/* + * bpf_get_socket_uid + * + * + * Returns + * The owner UID of the socket associated to *skb*. If the socket + * is **NULL**, or if it is not a full socket (i.e. if it is a + * time-wait or a request socket instead), **overflowuid** value + * is returned (note that **overflowuid** might also be the actual + * UID value for the socket). + */ +static __u32 (*bpf_get_socket_uid)(struct __sk_buff *skb) = (void *) 47; + +/* + * bpf_set_hash + * + * Set the full hash for *skb* (set the field *skb*\ **->hash**) + * to value *hash*. + * + * Returns + * 0 + */ +static long (*bpf_set_hash)(struct __sk_buff *skb, __u32 hash) = (void *) 48; + +/* + * bpf_setsockopt + * + * Emulate a call to **setsockopt()** on the socket associated to + * *bpf_socket*, which must be a full socket. The *level* at + * which the option resides and the name *optname* of the option + * must be specified, see **setsockopt(2)** for more information. + * The option value of length *optlen* is pointed by *optval*. + * + * *bpf_socket* should be one of the following: + * + * * **struct bpf_sock_ops** for **BPF_PROG_TYPE_SOCK_OPS**. + * * **struct bpf_sock_addr** for **BPF_CGROUP_INET4_CONNECT** + * and **BPF_CGROUP_INET6_CONNECT**. + * + * This helper actually implements a subset of **setsockopt()**. + * It supports the following *level*\ s: + * + * * **SOL_SOCKET**, which supports the following *optname*\ s: + * **SO_RCVBUF**, **SO_SNDBUF**, **SO_MAX_PACING_RATE**, + * **SO_PRIORITY**, **SO_RCVLOWAT**, **SO_MARK**, + * **SO_BINDTODEVICE**, **SO_KEEPALIVE**. + * * **IPPROTO_TCP**, which supports the following *optname*\ s: + * **TCP_CONGESTION**, **TCP_BPF_IW**, + * **TCP_BPF_SNDCWND_CLAMP**, **TCP_SAVE_SYN**, + * **TCP_KEEPIDLE**, **TCP_KEEPINTVL**, **TCP_KEEPCNT**, + * **TCP_SYNCNT**, **TCP_USER_TIMEOUT**. + * * **IPPROTO_IP**, which supports *optname* **IP_TOS**. + * * **IPPROTO_IPV6**, which supports *optname* **IPV6_TCLASS**. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_setsockopt)(void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *) 49; + +/* + * bpf_skb_adjust_room + * + * Grow or shrink the room for data in the packet associated to + * *skb* by *len_diff*, and according to the selected *mode*. + * + * By default, the helper will reset any offloaded checksum + * indicator of the skb to CHECKSUM_NONE. This can be avoided + * by the following flag: + * + * * **BPF_F_ADJ_ROOM_NO_CSUM_RESET**: Do not reset offloaded + * checksum data of the skb to CHECKSUM_NONE. + * + * There are two supported modes at this time: + * + * * **BPF_ADJ_ROOM_MAC**: Adjust room at the mac layer + * (room space is added or removed below the layer 2 header). + * + * * **BPF_ADJ_ROOM_NET**: Adjust room at the network layer + * (room space is added or removed below the layer 3 header). + * + * The following flags are supported at this time: + * + * * **BPF_F_ADJ_ROOM_FIXED_GSO**: Do not adjust gso_size. + * Adjusting mss in this way is not allowed for datagrams. + * + * * **BPF_F_ADJ_ROOM_ENCAP_L3_IPV4**, + * **BPF_F_ADJ_ROOM_ENCAP_L3_IPV6**: + * Any new space is reserved to hold a tunnel header. + * Configure skb offsets and other fields accordingly. + * + * * **BPF_F_ADJ_ROOM_ENCAP_L4_GRE**, + * **BPF_F_ADJ_ROOM_ENCAP_L4_UDP**: + * Use with ENCAP_L3 flags to further specify the tunnel type. + * + * * **BPF_F_ADJ_ROOM_ENCAP_L2**\ (*len*): + * Use with ENCAP_L3/L4 flags to further specify the tunnel + * type; *len* is the length of the inner MAC header. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_adjust_room)(struct __sk_buff *skb, __s32 len_diff, __u32 mode, __u64 flags) = (void *) 50; + +/* + * bpf_redirect_map + * + * Redirect the packet to the endpoint referenced by *map* at + * index *key*. Depending on its type, this *map* can contain + * references to net devices (for forwarding packets through other + * ports), or to CPUs (for redirecting XDP frames to another CPU; + * but this is only implemented for native XDP (with driver + * support) as of this writing). + * + * The lower two bits of *flags* are used as the return code if + * the map lookup fails. This is so that the return value can be + * one of the XDP program return codes up to **XDP_TX**, as chosen + * by the caller. Any higher bits in the *flags* argument must be + * unset. + * + * See also **bpf_redirect**\ (), which only supports redirecting + * to an ifindex, but doesn't require a map to do so. + * + * Returns + * **XDP_REDIRECT** on success, or the value of the two lower bits + * of the *flags* argument on error. + */ +static long (*bpf_redirect_map)(void *map, __u32 key, __u64 flags) = (void *) 51; + +/* + * bpf_sk_redirect_map + * + * Redirect the packet to the socket referenced by *map* (of type + * **BPF_MAP_TYPE_SOCKMAP**) at index *key*. Both ingress and + * egress interfaces can be used for redirection. The + * **BPF_F_INGRESS** value in *flags* is used to make the + * distinction (ingress path is selected if the flag is present, + * egress path otherwise). This is the only flag supported for now. + * + * Returns + * **SK_PASS** on success, or **SK_DROP** on error. + */ +static long (*bpf_sk_redirect_map)(struct __sk_buff *skb, void *map, __u32 key, __u64 flags) = (void *) 52; + +/* + * bpf_sock_map_update + * + * Add an entry to, or update a *map* referencing sockets. The + * *skops* is used as a new value for the entry associated to + * *key*. *flags* is one of: + * + * **BPF_NOEXIST** + * The entry for *key* must not exist in the map. + * **BPF_EXIST** + * The entry for *key* must already exist in the map. + * **BPF_ANY** + * No condition on the existence of the entry for *key*. + * + * If the *map* has eBPF programs (parser and verdict), those will + * be inherited by the socket being added. If the socket is + * already attached to eBPF programs, this results in an error. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_sock_map_update)(struct bpf_sock_ops *skops, void *map, void *key, __u64 flags) = (void *) 53; + +/* + * bpf_xdp_adjust_meta + * + * Adjust the address pointed by *xdp_md*\ **->data_meta** by + * *delta* (which can be positive or negative). Note that this + * operation modifies the address stored in *xdp_md*\ **->data**, + * so the latter must be loaded only after the helper has been + * called. + * + * The use of *xdp_md*\ **->data_meta** is optional and programs + * are not required to use it. The rationale is that when the + * packet is processed with XDP (e.g. as DoS filter), it is + * possible to push further meta data along with it before passing + * to the stack, and to give the guarantee that an ingress eBPF + * program attached as a TC classifier on the same device can pick + * this up for further post-processing. Since TC works with socket + * buffers, it remains possible to set from XDP the **mark** or + * **priority** pointers, or other pointers for the socket buffer. + * Having this scratch space generic and programmable allows for + * more flexibility as the user is free to store whatever meta + * data they need. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_xdp_adjust_meta)(struct xdp_md *xdp_md, int delta) = (void *) 54; + +/* + * bpf_perf_event_read_value + * + * Read the value of a perf event counter, and store it into *buf* + * of size *buf_size*. This helper relies on a *map* of type + * **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. The nature of the perf event + * counter is selected when *map* is updated with perf event file + * descriptors. The *map* is an array whose size is the number of + * available CPUs, and each cell contains a value relative to one + * CPU. The value to retrieve is indicated by *flags*, that + * contains the index of the CPU to look up, masked with + * **BPF_F_INDEX_MASK**. Alternatively, *flags* can be set to + * **BPF_F_CURRENT_CPU** to indicate that the value for the + * current CPU should be retrieved. + * + * This helper behaves in a way close to + * **bpf_perf_event_read**\ () helper, save that instead of + * just returning the value observed, it fills the *buf* + * structure. This allows for additional data to be retrieved: in + * particular, the enabled and running times (in *buf*\ + * **->enabled** and *buf*\ **->running**, respectively) are + * copied. In general, **bpf_perf_event_read_value**\ () is + * recommended over **bpf_perf_event_read**\ (), which has some + * ABI issues and provides fewer functionalities. + * + * These values are interesting, because hardware PMU (Performance + * Monitoring Unit) counters are limited resources. When there are + * more PMU based perf events opened than available counters, + * kernel will multiplex these events so each event gets certain + * percentage (but not all) of the PMU time. In case that + * multiplexing happens, the number of samples or counter value + * will not reflect the case compared to when no multiplexing + * occurs. This makes comparison between different runs difficult. + * Typically, the counter value should be normalized before + * comparing to other experiments. The usual normalization is done + * as follows. + * + * :: + * + * normalized_counter = counter * t_enabled / t_running + * + * Where t_enabled is the time enabled for event and t_running is + * the time running for event since last normalization. The + * enabled and running times are accumulated since the perf event + * open. To achieve scaling factor between two invocations of an + * eBPF program, users can use CPU id as the key (which is + * typical for perf array usage model) to remember the previous + * value and do the calculation inside the eBPF program. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_perf_event_read_value)(void *map, __u64 flags, struct bpf_perf_event_value *buf, __u32 buf_size) = (void *) 55; + +/* + * bpf_perf_prog_read_value + * + * For en eBPF program attached to a perf event, retrieve the + * value of the event counter associated to *ctx* and store it in + * the structure pointed by *buf* and of size *buf_size*. Enabled + * and running times are also stored in the structure (see + * description of helper **bpf_perf_event_read_value**\ () for + * more details). + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_perf_prog_read_value)(struct bpf_perf_event_data *ctx, struct bpf_perf_event_value *buf, __u32 buf_size) = (void *) 56; + +/* + * bpf_getsockopt + * + * Emulate a call to **getsockopt()** on the socket associated to + * *bpf_socket*, which must be a full socket. The *level* at + * which the option resides and the name *optname* of the option + * must be specified, see **getsockopt(2)** for more information. + * The retrieved value is stored in the structure pointed by + * *opval* and of length *optlen*. + * + * *bpf_socket* should be one of the following: + * + * * **struct bpf_sock_ops** for **BPF_PROG_TYPE_SOCK_OPS**. + * * **struct bpf_sock_addr** for **BPF_CGROUP_INET4_CONNECT** + * and **BPF_CGROUP_INET6_CONNECT**. + * + * This helper actually implements a subset of **getsockopt()**. + * It supports the following *level*\ s: + * + * * **IPPROTO_TCP**, which supports *optname* + * **TCP_CONGESTION**. + * * **IPPROTO_IP**, which supports *optname* **IP_TOS**. + * * **IPPROTO_IPV6**, which supports *optname* **IPV6_TCLASS**. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_getsockopt)(void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *) 57; + +/* + * bpf_override_return + * + * Used for error injection, this helper uses kprobes to override + * the return value of the probed function, and to set it to *rc*. + * The first argument is the context *regs* on which the kprobe + * works. + * + * This helper works by setting the PC (program counter) + * to an override function which is run in place of the original + * probed function. This means the probed function is not run at + * all. The replacement function just returns with the required + * value. + * + * This helper has security implications, and thus is subject to + * restrictions. It is only available if the kernel was compiled + * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration + * option, and in this case it only works on functions tagged with + * **ALLOW_ERROR_INJECTION** in the kernel code. + * + * Also, the helper is only available for the architectures having + * the CONFIG_FUNCTION_ERROR_INJECTION option. As of this writing, + * x86 architecture is the only one to support this feature. + * + * Returns + * 0 + */ +static long (*bpf_override_return)(struct pt_regs *regs, __u64 rc) = (void *) 58; + +/* + * bpf_sock_ops_cb_flags_set + * + * Attempt to set the value of the **bpf_sock_ops_cb_flags** field + * for the full TCP socket associated to *bpf_sock_ops* to + * *argval*. + * + * The primary use of this field is to determine if there should + * be calls to eBPF programs of type + * **BPF_PROG_TYPE_SOCK_OPS** at various points in the TCP + * code. A program of the same type can change its value, per + * connection and as necessary, when the connection is + * established. This field is directly accessible for reading, but + * this helper must be used for updates in order to return an + * error if an eBPF program tries to set a callback that is not + * supported in the current kernel. + * + * *argval* is a flag array which can combine these flags: + * + * * **BPF_SOCK_OPS_RTO_CB_FLAG** (retransmission time out) + * * **BPF_SOCK_OPS_RETRANS_CB_FLAG** (retransmission) + * * **BPF_SOCK_OPS_STATE_CB_FLAG** (TCP state change) + * * **BPF_SOCK_OPS_RTT_CB_FLAG** (every RTT) + * + * Therefore, this function can be used to clear a callback flag by + * setting the appropriate bit to zero. e.g. to disable the RTO + * callback: + * + * **bpf_sock_ops_cb_flags_set(bpf_sock,** + * **bpf_sock->bpf_sock_ops_cb_flags & ~BPF_SOCK_OPS_RTO_CB_FLAG)** + * + * Here are some examples of where one could call such eBPF + * program: + * + * * When RTO fires. + * * When a packet is retransmitted. + * * When the connection terminates. + * * When a packet is sent. + * * When a packet is received. + * + * Returns + * Code **-EINVAL** if the socket is not a full TCP socket; + * otherwise, a positive number containing the bits that could not + * be set is returned (which comes down to 0 if all bits were set + * as required). + */ +static long (*bpf_sock_ops_cb_flags_set)(struct bpf_sock_ops *bpf_sock, int argval) = (void *) 59; + +/* + * bpf_msg_redirect_map + * + * This helper is used in programs implementing policies at the + * socket level. If the message *msg* is allowed to pass (i.e. if + * the verdict eBPF program returns **SK_PASS**), redirect it to + * the socket referenced by *map* (of type + * **BPF_MAP_TYPE_SOCKMAP**) at index *key*. Both ingress and + * egress interfaces can be used for redirection. The + * **BPF_F_INGRESS** value in *flags* is used to make the + * distinction (ingress path is selected if the flag is present, + * egress path otherwise). This is the only flag supported for now. + * + * Returns + * **SK_PASS** on success, or **SK_DROP** on error. + */ +static long (*bpf_msg_redirect_map)(struct sk_msg_md *msg, void *map, __u32 key, __u64 flags) = (void *) 60; + +/* + * bpf_msg_apply_bytes + * + * For socket policies, apply the verdict of the eBPF program to + * the next *bytes* (number of bytes) of message *msg*. + * + * For example, this helper can be used in the following cases: + * + * * A single **sendmsg**\ () or **sendfile**\ () system call + * contains multiple logical messages that the eBPF program is + * supposed to read and for which it should apply a verdict. + * * An eBPF program only cares to read the first *bytes* of a + * *msg*. If the message has a large payload, then setting up + * and calling the eBPF program repeatedly for all bytes, even + * though the verdict is already known, would create unnecessary + * overhead. + * + * When called from within an eBPF program, the helper sets a + * counter internal to the BPF infrastructure, that is used to + * apply the last verdict to the next *bytes*. If *bytes* is + * smaller than the current data being processed from a + * **sendmsg**\ () or **sendfile**\ () system call, the first + * *bytes* will be sent and the eBPF program will be re-run with + * the pointer for start of data pointing to byte number *bytes* + * **+ 1**. If *bytes* is larger than the current data being + * processed, then the eBPF verdict will be applied to multiple + * **sendmsg**\ () or **sendfile**\ () calls until *bytes* are + * consumed. + * + * Note that if a socket closes with the internal counter holding + * a non-zero value, this is not a problem because data is not + * being buffered for *bytes* and is sent as it is received. + * + * Returns + * 0 + */ +static long (*bpf_msg_apply_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) 61; + +/* + * bpf_msg_cork_bytes + * + * For socket policies, prevent the execution of the verdict eBPF + * program for message *msg* until *bytes* (byte number) have been + * accumulated. + * + * This can be used when one needs a specific number of bytes + * before a verdict can be assigned, even if the data spans + * multiple **sendmsg**\ () or **sendfile**\ () calls. The extreme + * case would be a user calling **sendmsg**\ () repeatedly with + * 1-byte long message segments. Obviously, this is bad for + * performance, but it is still valid. If the eBPF program needs + * *bytes* bytes to validate a header, this helper can be used to + * prevent the eBPF program to be called again until *bytes* have + * been accumulated. + * + * Returns + * 0 + */ +static long (*bpf_msg_cork_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) 62; + +/* + * bpf_msg_pull_data + * + * For socket policies, pull in non-linear data from user space + * for *msg* and set pointers *msg*\ **->data** and *msg*\ + * **->data_end** to *start* and *end* bytes offsets into *msg*, + * respectively. + * + * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a + * *msg* it can only parse data that the (**data**, **data_end**) + * pointers have already consumed. For **sendmsg**\ () hooks this + * is likely the first scatterlist element. But for calls relying + * on the **sendpage** handler (e.g. **sendfile**\ ()) this will + * be the range (**0**, **0**) because the data is shared with + * user space and by default the objective is to avoid allowing + * user space to modify data while (or after) eBPF verdict is + * being decided. This helper can be used to pull in data and to + * set the start and end pointer to given values. Data will be + * copied if necessary (i.e. if data was not linear and if start + * and end pointers do not point to the same chunk). + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * All values for *flags* are reserved for future usage, and must + * be left at zero. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_msg_pull_data)(struct sk_msg_md *msg, __u32 start, __u32 end, __u64 flags) = (void *) 63; + +/* + * bpf_bind + * + * Bind the socket associated to *ctx* to the address pointed by + * *addr*, of length *addr_len*. This allows for making outgoing + * connection from the desired IP address, which can be useful for + * example when all processes inside a cgroup should use one + * single IP address on a host that has multiple IP configured. + * + * This helper works for IPv4 and IPv6, TCP and UDP sockets. The + * domain (*addr*\ **->sa_family**) must be **AF_INET** (or + * **AF_INET6**). It's advised to pass zero port (**sin_port** + * or **sin6_port**) which triggers IP_BIND_ADDRESS_NO_PORT-like + * behavior and lets the kernel efficiently pick up an unused + * port as long as 4-tuple is unique. Passing non-zero port might + * lead to degraded performance. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_bind)(struct bpf_sock_addr *ctx, struct sockaddr *addr, int addr_len) = (void *) 64; + +/* + * bpf_xdp_adjust_tail + * + * Adjust (move) *xdp_md*\ **->data_end** by *delta* bytes. It is + * possible to both shrink and grow the packet tail. + * Shrink done via *delta* being a negative integer. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_xdp_adjust_tail)(struct xdp_md *xdp_md, int delta) = (void *) 65; + +/* + * bpf_skb_get_xfrm_state + * + * Retrieve the XFRM state (IP transform framework, see also + * **ip-xfrm(8)**) at *index* in XFRM "security path" for *skb*. + * + * The retrieved value is stored in the **struct bpf_xfrm_state** + * pointed by *xfrm_state* and of length *size*. + * + * All values for *flags* are reserved for future usage, and must + * be left at zero. + * + * This helper is available only if the kernel was compiled with + * **CONFIG_XFRM** configuration option. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_get_xfrm_state)(struct __sk_buff *skb, __u32 index, struct bpf_xfrm_state *xfrm_state, __u32 size, __u64 flags) = (void *) 66; + +/* + * bpf_get_stack + * + * Return a user or a kernel stack in bpf program provided buffer. + * To achieve this, the helper needs *ctx*, which is a pointer + * to the context on which the tracing program is executed. + * To store the stacktrace, the bpf program provides *buf* with + * a nonnegative *size*. + * + * The last argument, *flags*, holds the number of stack frames to + * skip (from 0 to 255), masked with + * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set + * the following flags: + * + * **BPF_F_USER_STACK** + * Collect a user space stack instead of a kernel stack. + * **BPF_F_USER_BUILD_ID** + * Collect buildid+offset instead of ips for user stack, + * only valid if **BPF_F_USER_STACK** is also specified. + * + * **bpf_get_stack**\ () can collect up to + * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject + * to sufficient large buffer size. Note that + * this limit can be controlled with the **sysctl** program, and + * that it should be manually increased in order to profile long + * user stacks (such as stacks for Java programs). To do so, use: + * + * :: + * + * # sysctl kernel.perf_event_max_stack= + * + * Returns + * A non-negative value equal to or less than *size* on success, + * or a negative error in case of failure. + */ +static long (*bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (void *) 67; + +/* + * bpf_skb_load_bytes_relative + * + * This helper is similar to **bpf_skb_load_bytes**\ () in that + * it provides an easy way to load *len* bytes from *offset* + * from the packet associated to *skb*, into the buffer pointed + * by *to*. The difference to **bpf_skb_load_bytes**\ () is that + * a fifth argument *start_header* exists in order to select a + * base offset to start from. *start_header* can be one of: + * + * **BPF_HDR_START_MAC** + * Base offset to load data from is *skb*'s mac header. + * **BPF_HDR_START_NET** + * Base offset to load data from is *skb*'s network header. + * + * In general, "direct packet access" is the preferred method to + * access packet data, however, this helper is in particular useful + * in socket filters where *skb*\ **->data** does not always point + * to the start of the mac header and where "direct packet access" + * is not available. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_load_bytes_relative)(const void *skb, __u32 offset, void *to, __u32 len, __u32 start_header) = (void *) 68; + +/* + * bpf_fib_lookup + * + * Do FIB lookup in kernel tables using parameters in *params*. + * If lookup is successful and result shows packet is to be + * forwarded, the neighbor tables are searched for the nexthop. + * If successful (ie., FIB lookup shows forwarding and nexthop + * is resolved), the nexthop address is returned in ipv4_dst + * or ipv6_dst based on family, smac is set to mac address of + * egress device, dmac is set to nexthop mac address, rt_metric + * is set to metric from route (IPv4/IPv6 only), and ifindex + * is set to the device index of the nexthop from the FIB lookup. + * + * *plen* argument is the size of the passed in struct. + * *flags* argument can be a combination of one or more of the + * following values: + * + * **BPF_FIB_LOOKUP_DIRECT** + * Do a direct table lookup vs full lookup using FIB + * rules. + * **BPF_FIB_LOOKUP_OUTPUT** + * Perform lookup from an egress perspective (default is + * ingress). + * + * *ctx* is either **struct xdp_md** for XDP programs or + * **struct sk_buff** tc cls_act programs. + * + * Returns + * * < 0 if any input argument is invalid + * * 0 on success (packet is forwarded, nexthop neighbor exists) + * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the + * packet is not forwarded or needs assist from full stack + */ +static long (*bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, int plen, __u32 flags) = (void *) 69; + +/* + * bpf_sock_hash_update + * + * Add an entry to, or update a sockhash *map* referencing sockets. + * The *skops* is used as a new value for the entry associated to + * *key*. *flags* is one of: + * + * **BPF_NOEXIST** + * The entry for *key* must not exist in the map. + * **BPF_EXIST** + * The entry for *key* must already exist in the map. + * **BPF_ANY** + * No condition on the existence of the entry for *key*. + * + * If the *map* has eBPF programs (parser and verdict), those will + * be inherited by the socket being added. If the socket is + * already attached to eBPF programs, this results in an error. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_sock_hash_update)(struct bpf_sock_ops *skops, void *map, void *key, __u64 flags) = (void *) 70; + +/* + * bpf_msg_redirect_hash + * + * This helper is used in programs implementing policies at the + * socket level. If the message *msg* is allowed to pass (i.e. if + * the verdict eBPF program returns **SK_PASS**), redirect it to + * the socket referenced by *map* (of type + * **BPF_MAP_TYPE_SOCKHASH**) using hash *key*. Both ingress and + * egress interfaces can be used for redirection. The + * **BPF_F_INGRESS** value in *flags* is used to make the + * distinction (ingress path is selected if the flag is present, + * egress path otherwise). This is the only flag supported for now. + * + * Returns + * **SK_PASS** on success, or **SK_DROP** on error. + */ +static long (*bpf_msg_redirect_hash)(struct sk_msg_md *msg, void *map, void *key, __u64 flags) = (void *) 71; + +/* + * bpf_sk_redirect_hash + * + * This helper is used in programs implementing policies at the + * skb socket level. If the sk_buff *skb* is allowed to pass (i.e. + * if the verdeict eBPF program returns **SK_PASS**), redirect it + * to the socket referenced by *map* (of type + * **BPF_MAP_TYPE_SOCKHASH**) using hash *key*. Both ingress and + * egress interfaces can be used for redirection. The + * **BPF_F_INGRESS** value in *flags* is used to make the + * distinction (ingress path is selected if the flag is present, + * egress otherwise). This is the only flag supported for now. + * + * Returns + * **SK_PASS** on success, or **SK_DROP** on error. + */ +static long (*bpf_sk_redirect_hash)(struct __sk_buff *skb, void *map, void *key, __u64 flags) = (void *) 72; + +/* + * bpf_lwt_push_encap + * + * Encapsulate the packet associated to *skb* within a Layer 3 + * protocol header. This header is provided in the buffer at + * address *hdr*, with *len* its size in bytes. *type* indicates + * the protocol of the header and can be one of: + * + * **BPF_LWT_ENCAP_SEG6** + * IPv6 encapsulation with Segment Routing Header + * (**struct ipv6_sr_hdr**). *hdr* only contains the SRH, + * the IPv6 header is computed by the kernel. + * **BPF_LWT_ENCAP_SEG6_INLINE** + * Only works if *skb* contains an IPv6 packet. Insert a + * Segment Routing Header (**struct ipv6_sr_hdr**) inside + * the IPv6 header. + * **BPF_LWT_ENCAP_IP** + * IP encapsulation (GRE/GUE/IPIP/etc). The outer header + * must be IPv4 or IPv6, followed by zero or more + * additional headers, up to **LWT_BPF_MAX_HEADROOM** + * total bytes in all prepended headers. Please note that + * if **skb_is_gso**\ (*skb*) is true, no more than two + * headers can be prepended, and the inner header, if + * present, should be either GRE or UDP/GUE. + * + * **BPF_LWT_ENCAP_SEG6**\ \* types can be called by BPF programs + * of type **BPF_PROG_TYPE_LWT_IN**; **BPF_LWT_ENCAP_IP** type can + * be called by bpf programs of types **BPF_PROG_TYPE_LWT_IN** and + * **BPF_PROG_TYPE_LWT_XMIT**. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_lwt_push_encap)(struct __sk_buff *skb, __u32 type, void *hdr, __u32 len) = (void *) 73; + +/* + * bpf_lwt_seg6_store_bytes + * + * Store *len* bytes from address *from* into the packet + * associated to *skb*, at *offset*. Only the flags, tag and TLVs + * inside the outermost IPv6 Segment Routing Header can be + * modified through this helper. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_lwt_seg6_store_bytes)(struct __sk_buff *skb, __u32 offset, const void *from, __u32 len) = (void *) 74; + +/* + * bpf_lwt_seg6_adjust_srh + * + * Adjust the size allocated to TLVs in the outermost IPv6 + * Segment Routing Header contained in the packet associated to + * *skb*, at position *offset* by *delta* bytes. Only offsets + * after the segments are accepted. *delta* can be as well + * positive (growing) as negative (shrinking). + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_lwt_seg6_adjust_srh)(struct __sk_buff *skb, __u32 offset, __s32 delta) = (void *) 75; + +/* + * bpf_lwt_seg6_action + * + * Apply an IPv6 Segment Routing action of type *action* to the + * packet associated to *skb*. Each action takes a parameter + * contained at address *param*, and of length *param_len* bytes. + * *action* can be one of: + * + * **SEG6_LOCAL_ACTION_END_X** + * End.X action: Endpoint with Layer-3 cross-connect. + * Type of *param*: **struct in6_addr**. + * **SEG6_LOCAL_ACTION_END_T** + * End.T action: Endpoint with specific IPv6 table lookup. + * Type of *param*: **int**. + * **SEG6_LOCAL_ACTION_END_B6** + * End.B6 action: Endpoint bound to an SRv6 policy. + * Type of *param*: **struct ipv6_sr_hdr**. + * **SEG6_LOCAL_ACTION_END_B6_ENCAP** + * End.B6.Encap action: Endpoint bound to an SRv6 + * encapsulation policy. + * Type of *param*: **struct ipv6_sr_hdr**. + * + * A call to this helper is susceptible to change the underlying + * packet buffer. Therefore, at load time, all checks on pointers + * previously done by the verifier are invalidated and must be + * performed again, if the helper is used in combination with + * direct packet access. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_lwt_seg6_action)(struct __sk_buff *skb, __u32 action, void *param, __u32 param_len) = (void *) 76; + +/* + * bpf_rc_repeat + * + * This helper is used in programs implementing IR decoding, to + * report a successfully decoded repeat key message. This delays + * the generation of a key up event for previously generated + * key down event. + * + * Some IR protocols like NEC have a special IR message for + * repeating last button, for when a button is held down. + * + * The *ctx* should point to the lirc sample as passed into + * the program. + * + * This helper is only available is the kernel was compiled with + * the **CONFIG_BPF_LIRC_MODE2** configuration option set to + * "**y**". + * + * Returns + * 0 + */ +static long (*bpf_rc_repeat)(void *ctx) = (void *) 77; + +/* + * bpf_rc_keydown + * + * This helper is used in programs implementing IR decoding, to + * report a successfully decoded key press with *scancode*, + * *toggle* value in the given *protocol*. The scancode will be + * translated to a keycode using the rc keymap, and reported as + * an input key down event. After a period a key up event is + * generated. This period can be extended by calling either + * **bpf_rc_keydown**\ () again with the same values, or calling + * **bpf_rc_repeat**\ (). + * + * Some protocols include a toggle bit, in case the button was + * released and pressed again between consecutive scancodes. + * + * The *ctx* should point to the lirc sample as passed into + * the program. + * + * The *protocol* is the decoded protocol number (see + * **enum rc_proto** for some predefined values). + * + * This helper is only available is the kernel was compiled with + * the **CONFIG_BPF_LIRC_MODE2** configuration option set to + * "**y**". + * + * Returns + * 0 + */ +static long (*bpf_rc_keydown)(void *ctx, __u32 protocol, __u64 scancode, __u32 toggle) = (void *) 78; + +/* + * bpf_skb_cgroup_id + * + * Return the cgroup v2 id of the socket associated with the *skb*. + * This is roughly similar to the **bpf_get_cgroup_classid**\ () + * helper for cgroup v1 by providing a tag resp. identifier that + * can be matched on or used for map lookups e.g. to implement + * policy. The cgroup v2 id of a given path in the hierarchy is + * exposed in user space through the f_handle API in order to get + * to the same 64-bit id. + * + * This helper can be used on TC egress path, but not on ingress, + * and is available only if the kernel was compiled with the + * **CONFIG_SOCK_CGROUP_DATA** configuration option. + * + * Returns + * The id is returned or 0 in case the id could not be retrieved. + */ +static __u64 (*bpf_skb_cgroup_id)(struct __sk_buff *skb) = (void *) 79; + +/* + * bpf_get_current_cgroup_id + * + * + * Returns + * A 64-bit integer containing the current cgroup id based + * on the cgroup within which the current task is running. + */ +static __u64 (*bpf_get_current_cgroup_id)(void) = (void *) 80; + +/* + * bpf_get_local_storage + * + * Get the pointer to the local storage area. + * The type and the size of the local storage is defined + * by the *map* argument. + * The *flags* meaning is specific for each map type, + * and has to be 0 for cgroup local storage. + * + * Depending on the BPF program type, a local storage area + * can be shared between multiple instances of the BPF program, + * running simultaneously. + * + * A user should care about the synchronization by himself. + * For example, by using the **BPF_STX_XADD** instruction to alter + * the shared data. + * + * Returns + * A pointer to the local storage area. + */ +static void *(*bpf_get_local_storage)(void *map, __u64 flags) = (void *) 81; + +/* + * bpf_sk_select_reuseport + * + * Select a **SO_REUSEPORT** socket from a + * **BPF_MAP_TYPE_REUSEPORT_ARRAY** *map*. + * It checks the selected socket is matching the incoming + * request in the socket buffer. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_sk_select_reuseport)(struct sk_reuseport_md *reuse, void *map, void *key, __u64 flags) = (void *) 82; + +/* + * bpf_skb_ancestor_cgroup_id + * + * Return id of cgroup v2 that is ancestor of cgroup associated + * with the *skb* at the *ancestor_level*. The root cgroup is at + * *ancestor_level* zero and each step down the hierarchy + * increments the level. If *ancestor_level* == level of cgroup + * associated with *skb*, then return value will be same as that + * of **bpf_skb_cgroup_id**\ (). + * + * The helper is useful to implement policies based on cgroups + * that are upper in hierarchy than immediate cgroup associated + * with *skb*. + * + * The format of returned id and helper limitations are same as in + * **bpf_skb_cgroup_id**\ (). + * + * Returns + * The id is returned or 0 in case the id could not be retrieved. + */ +static __u64 (*bpf_skb_ancestor_cgroup_id)(struct __sk_buff *skb, int ancestor_level) = (void *) 83; + +/* + * bpf_sk_lookup_tcp + * + * Look for TCP socket matching *tuple*, optionally in a child + * network namespace *netns*. The return value must be checked, + * and if non-**NULL**, released via **bpf_sk_release**\ (). + * + * The *ctx* should point to the context of the program, such as + * the skb or socket (depending on the hook in use). This is used + * to determine the base network namespace for the lookup. + * + * *tuple_size* must be one of: + * + * **sizeof**\ (*tuple*\ **->ipv4**) + * Look for an IPv4 socket. + * **sizeof**\ (*tuple*\ **->ipv6**) + * Look for an IPv6 socket. + * + * If the *netns* is a negative signed 32-bit integer, then the + * socket lookup table in the netns associated with the *ctx* + * will be used. For the TC hooks, this is the netns of the device + * in the skb. For socket hooks, this is the netns of the socket. + * If *netns* is any other signed 32-bit value greater than or + * equal to zero then it specifies the ID of the netns relative to + * the netns associated with the *ctx*. *netns* values beyond the + * range of 32-bit integers are reserved for future use. + * + * All values for *flags* are reserved for future usage, and must + * be left at zero. + * + * This helper is available only if the kernel was compiled with + * **CONFIG_NET** configuration option. + * + * Returns + * Pointer to **struct bpf_sock**, or **NULL** in case of failure. + * For sockets with reuseport option, the **struct bpf_sock** + * result is from *reuse*\ **->socks**\ [] using the hash of the + * tuple. + */ +static struct bpf_sock *(*bpf_sk_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 84; + +/* + * bpf_sk_lookup_udp + * + * Look for UDP socket matching *tuple*, optionally in a child + * network namespace *netns*. The return value must be checked, + * and if non-**NULL**, released via **bpf_sk_release**\ (). + * + * The *ctx* should point to the context of the program, such as + * the skb or socket (depending on the hook in use). This is used + * to determine the base network namespace for the lookup. + * + * *tuple_size* must be one of: + * + * **sizeof**\ (*tuple*\ **->ipv4**) + * Look for an IPv4 socket. + * **sizeof**\ (*tuple*\ **->ipv6**) + * Look for an IPv6 socket. + * + * If the *netns* is a negative signed 32-bit integer, then the + * socket lookup table in the netns associated with the *ctx* + * will be used. For the TC hooks, this is the netns of the device + * in the skb. For socket hooks, this is the netns of the socket. + * If *netns* is any other signed 32-bit value greater than or + * equal to zero then it specifies the ID of the netns relative to + * the netns associated with the *ctx*. *netns* values beyond the + * range of 32-bit integers are reserved for future use. + * + * All values for *flags* are reserved for future usage, and must + * be left at zero. + * + * This helper is available only if the kernel was compiled with + * **CONFIG_NET** configuration option. + * + * Returns + * Pointer to **struct bpf_sock**, or **NULL** in case of failure. + * For sockets with reuseport option, the **struct bpf_sock** + * result is from *reuse*\ **->socks**\ [] using the hash of the + * tuple. + */ +static struct bpf_sock *(*bpf_sk_lookup_udp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 85; + +/* + * bpf_sk_release + * + * Release the reference held by *sock*. *sock* must be a + * non-**NULL** pointer that was returned from + * **bpf_sk_lookup_xxx**\ (). + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_sk_release)(struct bpf_sock *sock) = (void *) 86; + +/* + * bpf_map_push_elem + * + * Push an element *value* in *map*. *flags* is one of: + * + * **BPF_EXIST** + * If the queue/stack is full, the oldest element is + * removed to make room for this. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_map_push_elem)(void *map, const void *value, __u64 flags) = (void *) 87; + +/* + * bpf_map_pop_elem + * + * Pop an element from *map*. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_map_pop_elem)(void *map, void *value) = (void *) 88; + +/* + * bpf_map_peek_elem + * + * Get an element from *map* without removing it. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_map_peek_elem)(void *map, void *value) = (void *) 89; + +/* + * bpf_msg_push_data + * + * For socket policies, insert *len* bytes into *msg* at offset + * *start*. + * + * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a + * *msg* it may want to insert metadata or options into the *msg*. + * This can later be read and used by any of the lower layer BPF + * hooks. + * + * This helper may fail if under memory pressure (a malloc + * fails) in these cases BPF programs will get an appropriate + * error and BPF programs will need to handle them. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_msg_push_data)(struct sk_msg_md *msg, __u32 start, __u32 len, __u64 flags) = (void *) 90; + +/* + * bpf_msg_pop_data + * + * Will remove *len* bytes from a *msg* starting at byte *start*. + * This may result in **ENOMEM** errors under certain situations if + * an allocation and copy are required due to a full ring buffer. + * However, the helper will try to avoid doing the allocation + * if possible. Other errors can occur if input parameters are + * invalid either due to *start* byte not being valid part of *msg* + * payload and/or *pop* value being to large. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_msg_pop_data)(struct sk_msg_md *msg, __u32 start, __u32 len, __u64 flags) = (void *) 91; + +/* + * bpf_rc_pointer_rel + * + * This helper is used in programs implementing IR decoding, to + * report a successfully decoded pointer movement. + * + * The *ctx* should point to the lirc sample as passed into + * the program. + * + * This helper is only available is the kernel was compiled with + * the **CONFIG_BPF_LIRC_MODE2** configuration option set to + * "**y**". + * + * Returns + * 0 + */ +static long (*bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = (void *) 92; + +/* + * bpf_spin_lock + * + * Acquire a spinlock represented by the pointer *lock*, which is + * stored as part of a value of a map. Taking the lock allows to + * safely update the rest of the fields in that value. The + * spinlock can (and must) later be released with a call to + * **bpf_spin_unlock**\ (\ *lock*\ ). + * + * Spinlocks in BPF programs come with a number of restrictions + * and constraints: + * + * * **bpf_spin_lock** objects are only allowed inside maps of + * types **BPF_MAP_TYPE_HASH** and **BPF_MAP_TYPE_ARRAY** (this + * list could be extended in the future). + * * BTF description of the map is mandatory. + * * The BPF program can take ONE lock at a time, since taking two + * or more could cause dead locks. + * * Only one **struct bpf_spin_lock** is allowed per map element. + * * When the lock is taken, calls (either BPF to BPF or helpers) + * are not allowed. + * * The **BPF_LD_ABS** and **BPF_LD_IND** instructions are not + * allowed inside a spinlock-ed region. + * * The BPF program MUST call **bpf_spin_unlock**\ () to release + * the lock, on all execution paths, before it returns. + * * The BPF program can access **struct bpf_spin_lock** only via + * the **bpf_spin_lock**\ () and **bpf_spin_unlock**\ () + * helpers. Loading or storing data into the **struct + * bpf_spin_lock** *lock*\ **;** field of a map is not allowed. + * * To use the **bpf_spin_lock**\ () helper, the BTF description + * of the map value must be a struct and have **struct + * bpf_spin_lock** *anyname*\ **;** field at the top level. + * Nested lock inside another struct is not allowed. + * * The **struct bpf_spin_lock** *lock* field in a map value must + * be aligned on a multiple of 4 bytes in that value. + * * Syscall with command **BPF_MAP_LOOKUP_ELEM** does not copy + * the **bpf_spin_lock** field to user space. + * * Syscall with command **BPF_MAP_UPDATE_ELEM**, or update from + * a BPF program, do not update the **bpf_spin_lock** field. + * * **bpf_spin_lock** cannot be on the stack or inside a + * networking packet (it can only be inside of a map values). + * * **bpf_spin_lock** is available to root only. + * * Tracing programs and socket filter programs cannot use + * **bpf_spin_lock**\ () due to insufficient preemption checks + * (but this may change in the future). + * * **bpf_spin_lock** is not allowed in inner maps of map-in-map. + * + * Returns + * 0 + */ +static long (*bpf_spin_lock)(struct bpf_spin_lock *lock) = (void *) 93; + +/* + * bpf_spin_unlock + * + * Release the *lock* previously locked by a call to + * **bpf_spin_lock**\ (\ *lock*\ ). + * + * Returns + * 0 + */ +static long (*bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void *) 94; + +/* + * bpf_sk_fullsock + * + * This helper gets a **struct bpf_sock** pointer such + * that all the fields in this **bpf_sock** can be accessed. + * + * Returns + * A **struct bpf_sock** pointer on success, or **NULL** in + * case of failure. + */ +static struct bpf_sock *(*bpf_sk_fullsock)(struct bpf_sock *sk) = (void *) 95; + +/* + * bpf_tcp_sock + * + * This helper gets a **struct bpf_tcp_sock** pointer from a + * **struct bpf_sock** pointer. + * + * Returns + * A **struct bpf_tcp_sock** pointer on success, or **NULL** in + * case of failure. + */ +static struct bpf_tcp_sock *(*bpf_tcp_sock)(struct bpf_sock *sk) = (void *) 96; + +/* + * bpf_skb_ecn_set_ce + * + * Set ECN (Explicit Congestion Notification) field of IP header + * to **CE** (Congestion Encountered) if current value is **ECT** + * (ECN Capable Transport). Otherwise, do nothing. Works with IPv6 + * and IPv4. + * + * Returns + * 1 if the **CE** flag is set (either by the current helper call + * or because it was already present), 0 if it is not set. + */ +static long (*bpf_skb_ecn_set_ce)(struct __sk_buff *skb) = (void *) 97; + +/* + * bpf_get_listener_sock + * + * Return a **struct bpf_sock** pointer in **TCP_LISTEN** state. + * **bpf_sk_release**\ () is unnecessary and not allowed. + * + * Returns + * A **struct bpf_sock** pointer on success, or **NULL** in + * case of failure. + */ +static struct bpf_sock *(*bpf_get_listener_sock)(struct bpf_sock *sk) = (void *) 98; + +/* + * bpf_skc_lookup_tcp + * + * Look for TCP socket matching *tuple*, optionally in a child + * network namespace *netns*. The return value must be checked, + * and if non-**NULL**, released via **bpf_sk_release**\ (). + * + * This function is identical to **bpf_sk_lookup_tcp**\ (), except + * that it also returns timewait or request sockets. Use + * **bpf_sk_fullsock**\ () or **bpf_tcp_sock**\ () to access the + * full structure. + * + * This helper is available only if the kernel was compiled with + * **CONFIG_NET** configuration option. + * + * Returns + * Pointer to **struct bpf_sock**, or **NULL** in case of failure. + * For sockets with reuseport option, the **struct bpf_sock** + * result is from *reuse*\ **->socks**\ [] using the hash of the + * tuple. + */ +static struct bpf_sock *(*bpf_skc_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 99; + +/* + * bpf_tcp_check_syncookie + * + * Check whether *iph* and *th* contain a valid SYN cookie ACK for + * the listening socket in *sk*. + * + * *iph* points to the start of the IPv4 or IPv6 header, while + * *iph_len* contains **sizeof**\ (**struct iphdr**) or + * **sizeof**\ (**struct ip6hdr**). + * + * *th* points to the start of the TCP header, while *th_len* + * contains **sizeof**\ (**struct tcphdr**). + * + * Returns + * 0 if *iph* and *th* are a valid SYN cookie ACK, or a negative + * error otherwise. + */ +static long (*bpf_tcp_check_syncookie)(struct bpf_sock *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *) 100; + +/* + * bpf_sysctl_get_name + * + * Get name of sysctl in /proc/sys/ and copy it into provided by + * program buffer *buf* of size *buf_len*. + * + * The buffer is always NUL terminated, unless it's zero-sized. + * + * If *flags* is zero, full name (e.g. "net/ipv4/tcp_mem") is + * copied. Use **BPF_F_SYSCTL_BASE_NAME** flag to copy base name + * only (e.g. "tcp_mem"). + * + * Returns + * Number of character copied (not including the trailing NUL). + * + * **-E2BIG** if the buffer wasn't big enough (*buf* will contain + * truncated name in this case). + */ +static long (*bpf_sysctl_get_name)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len, __u64 flags) = (void *) 101; + +/* + * bpf_sysctl_get_current_value + * + * Get current value of sysctl as it is presented in /proc/sys + * (incl. newline, etc), and copy it as a string into provided + * by program buffer *buf* of size *buf_len*. + * + * The whole value is copied, no matter what file position user + * space issued e.g. sys_read at. + * + * The buffer is always NUL terminated, unless it's zero-sized. + * + * Returns + * Number of character copied (not including the trailing NUL). + * + * **-E2BIG** if the buffer wasn't big enough (*buf* will contain + * truncated name in this case). + * + * **-EINVAL** if current value was unavailable, e.g. because + * sysctl is uninitialized and read returns -EIO for it. + */ +static long (*bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len) = (void *) 102; + +/* + * bpf_sysctl_get_new_value + * + * Get new value being written by user space to sysctl (before + * the actual write happens) and copy it as a string into + * provided by program buffer *buf* of size *buf_len*. + * + * User space may write new value at file position > 0. + * + * The buffer is always NUL terminated, unless it's zero-sized. + * + * Returns + * Number of character copied (not including the trailing NUL). + * + * **-E2BIG** if the buffer wasn't big enough (*buf* will contain + * truncated name in this case). + * + * **-EINVAL** if sysctl is being read. + */ +static long (*bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len) = (void *) 103; + +/* + * bpf_sysctl_set_new_value + * + * Override new value being written by user space to sysctl with + * value provided by program in buffer *buf* of size *buf_len*. + * + * *buf* should contain a string in same form as provided by user + * space on sysctl write. + * + * User space may write new value at file position > 0. To override + * the whole sysctl value file position should be set to zero. + * + * Returns + * 0 on success. + * + * **-E2BIG** if the *buf_len* is too big. + * + * **-EINVAL** if sysctl is being read. + */ +static long (*bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, const char *buf, unsigned long buf_len) = (void *) 104; + +/* + * bpf_strtol + * + * Convert the initial part of the string from buffer *buf* of + * size *buf_len* to a long integer according to the given base + * and save the result in *res*. + * + * The string may begin with an arbitrary amount of white space + * (as determined by **isspace**\ (3)) followed by a single + * optional '**-**' sign. + * + * Five least significant bits of *flags* encode base, other bits + * are currently unused. + * + * Base must be either 8, 10, 16 or 0 to detect it automatically + * similar to user space **strtol**\ (3). + * + * Returns + * Number of characters consumed on success. Must be positive but + * no more than *buf_len*. + * + * **-EINVAL** if no valid digits were found or unsupported base + * was provided. + * + * **-ERANGE** if resulting value was out of range. + */ +static long (*bpf_strtol)(const char *buf, unsigned long buf_len, __u64 flags, long *res) = (void *) 105; + +/* + * bpf_strtoul + * + * Convert the initial part of the string from buffer *buf* of + * size *buf_len* to an unsigned long integer according to the + * given base and save the result in *res*. + * + * The string may begin with an arbitrary amount of white space + * (as determined by **isspace**\ (3)). + * + * Five least significant bits of *flags* encode base, other bits + * are currently unused. + * + * Base must be either 8, 10, 16 or 0 to detect it automatically + * similar to user space **strtoul**\ (3). + * + * Returns + * Number of characters consumed on success. Must be positive but + * no more than *buf_len*. + * + * **-EINVAL** if no valid digits were found or unsupported base + * was provided. + * + * **-ERANGE** if resulting value was out of range. + */ +static long (*bpf_strtoul)(const char *buf, unsigned long buf_len, __u64 flags, unsigned long *res) = (void *) 106; + +/* + * bpf_sk_storage_get + * + * Get a bpf-local-storage from a *sk*. + * + * Logically, it could be thought of getting the value from + * a *map* with *sk* as the **key**. From this + * perspective, the usage is not much different from + * **bpf_map_lookup_elem**\ (*map*, **&**\ *sk*) except this + * helper enforces the key must be a full socket and the map must + * be a **BPF_MAP_TYPE_SK_STORAGE** also. + * + * Underneath, the value is stored locally at *sk* instead of + * the *map*. The *map* is used as the bpf-local-storage + * "type". The bpf-local-storage "type" (i.e. the *map*) is + * searched against all bpf-local-storages residing at *sk*. + * + * An optional *flags* (**BPF_SK_STORAGE_GET_F_CREATE**) can be + * used such that a new bpf-local-storage will be + * created if one does not exist. *value* can be used + * together with **BPF_SK_STORAGE_GET_F_CREATE** to specify + * the initial value of a bpf-local-storage. If *value* is + * **NULL**, the new bpf-local-storage will be zero initialized. + * + * Returns + * A bpf-local-storage pointer is returned on success. + * + * **NULL** if not found or there was an error in adding + * a new bpf-local-storage. + */ +static void *(*bpf_sk_storage_get)(void *map, struct bpf_sock *sk, void *value, __u64 flags) = (void *) 107; + +/* + * bpf_sk_storage_delete + * + * Delete a bpf-local-storage from a *sk*. + * + * Returns + * 0 on success. + * + * **-ENOENT** if the bpf-local-storage cannot be found. + */ +static long (*bpf_sk_storage_delete)(void *map, struct bpf_sock *sk) = (void *) 108; + +/* + * bpf_send_signal + * + * Send signal *sig* to the process of the current task. + * The signal may be delivered to any of this process's threads. + * + * Returns + * 0 on success or successfully queued. + * + * **-EBUSY** if work queue under nmi is full. + * + * **-EINVAL** if *sig* is invalid. + * + * **-EPERM** if no permission to send the *sig*. + * + * **-EAGAIN** if bpf program can try again. + */ +static long (*bpf_send_signal)(__u32 sig) = (void *) 109; + +/* + * bpf_tcp_gen_syncookie + * + * Try to issue a SYN cookie for the packet with corresponding + * IP/TCP headers, *iph* and *th*, on the listening socket in *sk*. + * + * *iph* points to the start of the IPv4 or IPv6 header, while + * *iph_len* contains **sizeof**\ (**struct iphdr**) or + * **sizeof**\ (**struct ip6hdr**). + * + * *th* points to the start of the TCP header, while *th_len* + * contains the length of the TCP header. + * + * Returns + * On success, lower 32 bits hold the generated SYN cookie in + * followed by 16 bits which hold the MSS value for that cookie, + * and the top 16 bits are unused. + * + * On failure, the returned value is one of the following: + * + * **-EINVAL** SYN cookie cannot be issued due to error + * + * **-ENOENT** SYN cookie should not be issued (no SYN flood) + * + * **-EOPNOTSUPP** kernel configuration does not enable SYN cookies + * + * **-EPROTONOSUPPORT** IP packet version is not 4 or 6 + */ +static __s64 (*bpf_tcp_gen_syncookie)(struct bpf_sock *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *) 110; + +/* + * bpf_skb_output + * + * Write raw *data* blob into a special BPF perf event held by + * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf + * event must have the following attributes: **PERF_SAMPLE_RAW** + * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and + * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. + * + * The *flags* are used to indicate the index in *map* for which + * the value must be put, masked with **BPF_F_INDEX_MASK**. + * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** + * to indicate that the index of the current CPU core should be + * used. + * + * The value to write, of *size*, is passed through eBPF stack and + * pointed by *data*. + * + * *ctx* is a pointer to in-kernel struct sk_buff. + * + * This helper is similar to **bpf_perf_event_output**\ () but + * restricted to raw_tracepoint bpf programs. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_skb_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 111; + +/* + * bpf_probe_read_user + * + * Safely attempt to read *size* bytes from user space address + * *unsafe_ptr* and store the data in *dst*. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_probe_read_user)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 112; + +/* + * bpf_probe_read_kernel + * + * Safely attempt to read *size* bytes from kernel space address + * *unsafe_ptr* and store the data in *dst*. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_probe_read_kernel)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 113; + +/* + * bpf_probe_read_user_str + * + * Copy a NUL terminated string from an unsafe user address + * *unsafe_ptr* to *dst*. The *size* should include the + * terminating NUL byte. In case the string length is smaller than + * *size*, the target is not padded with further NUL bytes. If the + * string length is larger than *size*, just *size*-1 bytes are + * copied and the last byte is set to NUL. + * + * On success, the length of the copied string is returned. This + * makes this helper useful in tracing programs for reading + * strings, and more importantly to get its length at runtime. See + * the following snippet: + * + * :: + * + * SEC("kprobe/sys_open") + * void bpf_sys_open(struct pt_regs *ctx) + * { + * char buf[PATHLEN]; // PATHLEN is defined to 256 + * int res = bpf_probe_read_user_str(buf, sizeof(buf), + * ctx->di); + * + * // Consume buf, for example push it to + * // userspace via bpf_perf_event_output(); we + * // can use res (the string length) as event + * // size, after checking its boundaries. + * } + * + * In comparison, using **bpf_probe_read_user**\ () helper here + * instead to read the string would require to estimate the length + * at compile time, and would often result in copying more memory + * than necessary. + * + * Another useful use case is when parsing individual process + * arguments or individual environment variables navigating + * *current*\ **->mm->arg_start** and *current*\ + * **->mm->env_start**: using this helper and the return value, + * one can quickly iterate at the right offset of the memory area. + * + * Returns + * On success, the strictly positive length of the string, + * including the trailing NUL character. On error, a negative + * value. + */ +static long (*bpf_probe_read_user_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 114; + +/* + * bpf_probe_read_kernel_str + * + * Copy a NUL terminated string from an unsafe kernel address *unsafe_ptr* + * to *dst*. Same semantics as with **bpf_probe_read_user_str**\ () apply. + * + * Returns + * On success, the strictly positive length of the string, including + * the trailing NUL character. On error, a negative value. + */ +static long (*bpf_probe_read_kernel_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 115; + +/* + * bpf_tcp_send_ack + * + * Send out a tcp-ack. *tp* is the in-kernel struct **tcp_sock**. + * *rcv_nxt* is the ack_seq to be sent out. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_tcp_send_ack)(void *tp, __u32 rcv_nxt) = (void *) 116; + +/* + * bpf_send_signal_thread + * + * Send signal *sig* to the thread corresponding to the current task. + * + * Returns + * 0 on success or successfully queued. + * + * **-EBUSY** if work queue under nmi is full. + * + * **-EINVAL** if *sig* is invalid. + * + * **-EPERM** if no permission to send the *sig*. + * + * **-EAGAIN** if bpf program can try again. + */ +static long (*bpf_send_signal_thread)(__u32 sig) = (void *) 117; + +/* + * bpf_jiffies64 + * + * Obtain the 64bit jiffies + * + * Returns + * The 64 bit jiffies + */ +static __u64 (*bpf_jiffies64)(void) = (void *) 118; + +/* + * bpf_read_branch_records + * + * For an eBPF program attached to a perf event, retrieve the + * branch records (**struct perf_branch_entry**) associated to *ctx* + * and store it in the buffer pointed by *buf* up to size + * *size* bytes. + * + * Returns + * On success, number of bytes written to *buf*. On error, a + * negative value. + * + * The *flags* can be set to **BPF_F_GET_BRANCH_RECORDS_SIZE** to + * instead return the number of bytes required to store all the + * branch entries. If this flag is set, *buf* may be NULL. + * + * **-EINVAL** if arguments invalid or **size** not a multiple + * of **sizeof**\ (**struct perf_branch_entry**\ ). + * + * **-ENOENT** if architecture does not support branch records. + */ +static long (*bpf_read_branch_records)(struct bpf_perf_event_data *ctx, void *buf, __u32 size, __u64 flags) = (void *) 119; + +/* + * bpf_get_ns_current_pid_tgid + * + * Returns 0 on success, values for *pid* and *tgid* as seen from the current + * *namespace* will be returned in *nsdata*. + * + * Returns + * 0 on success, or one of the following in case of failure: + * + * **-EINVAL** if dev and inum supplied don't match dev_t and inode number + * with nsfs of current task, or if dev conversion to dev_t lost high bits. + * + * **-ENOENT** if pidns does not exists for the current task. + */ +static long (*bpf_get_ns_current_pid_tgid)(__u64 dev, __u64 ino, struct bpf_pidns_info *nsdata, __u32 size) = (void *) 120; + +/* + * bpf_xdp_output + * + * Write raw *data* blob into a special BPF perf event held by + * *map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf + * event must have the following attributes: **PERF_SAMPLE_RAW** + * as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and + * **PERF_COUNT_SW_BPF_OUTPUT** as **config**. + * + * The *flags* are used to indicate the index in *map* for which + * the value must be put, masked with **BPF_F_INDEX_MASK**. + * Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU** + * to indicate that the index of the current CPU core should be + * used. + * + * The value to write, of *size*, is passed through eBPF stack and + * pointed by *data*. + * + * *ctx* is a pointer to in-kernel struct xdp_buff. + * + * This helper is similar to **bpf_perf_eventoutput**\ () but + * restricted to raw_tracepoint bpf programs. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_xdp_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 121; + +/* + * bpf_get_netns_cookie + * + * Retrieve the cookie (generated by the kernel) of the network + * namespace the input *ctx* is associated with. The network + * namespace cookie remains stable for its lifetime and provides + * a global identifier that can be assumed unique. If *ctx* is + * NULL, then the helper returns the cookie for the initial + * network namespace. The cookie itself is very similar to that + * of **bpf_get_socket_cookie**\ () helper, but for network + * namespaces instead of sockets. + * + * Returns + * A 8-byte long opaque number. + */ +static __u64 (*bpf_get_netns_cookie)(void *ctx) = (void *) 122; + +/* + * bpf_get_current_ancestor_cgroup_id + * + * Return id of cgroup v2 that is ancestor of the cgroup associated + * with the current task at the *ancestor_level*. The root cgroup + * is at *ancestor_level* zero and each step down the hierarchy + * increments the level. If *ancestor_level* == level of cgroup + * associated with the current task, then return value will be the + * same as that of **bpf_get_current_cgroup_id**\ (). + * + * The helper is useful to implement policies based on cgroups + * that are upper in hierarchy than immediate cgroup associated + * with the current task. + * + * The format of returned id and helper limitations are same as in + * **bpf_get_current_cgroup_id**\ (). + * + * Returns + * The id is returned or 0 in case the id could not be retrieved. + */ +static __u64 (*bpf_get_current_ancestor_cgroup_id)(int ancestor_level) = (void *) 123; + +/* + * bpf_sk_assign + * + * Helper is overloaded depending on BPF program type. This + * description applies to **BPF_PROG_TYPE_SCHED_CLS** and + * **BPF_PROG_TYPE_SCHED_ACT** programs. + * + * Assign the *sk* to the *skb*. When combined with appropriate + * routing configuration to receive the packet towards the socket, + * will cause *skb* to be delivered to the specified socket. + * Subsequent redirection of *skb* via **bpf_redirect**\ (), + * **bpf_clone_redirect**\ () or other methods outside of BPF may + * interfere with successful delivery to the socket. + * + * This operation is only valid from TC ingress path. + * + * The *flags* argument must be zero. + * + * Returns + * 0 on success, or a negative error in case of failure: + * + * **-EINVAL** if specified *flags* are not supported. + * + * **-ENOENT** if the socket is unavailable for assignment. + * + * **-ENETUNREACH** if the socket is unreachable (wrong netns). + * + * **-EOPNOTSUPP** if the operation is not supported, for example + * a call from outside of TC ingress. + * + * **-ESOCKTNOSUPPORT** if the socket type is not supported + * (reuseport). + */ +static long (*bpf_sk_assign)(void *ctx, struct bpf_sock *sk, __u64 flags) = (void *) 124; + +/* + * bpf_ktime_get_boot_ns + * + * Return the time elapsed since system boot, in nanoseconds. + * Does include the time the system was suspended. + * See: **clock_gettime**\ (**CLOCK_BOOTTIME**) + * + * Returns + * Current *ktime*. + */ +static __u64 (*bpf_ktime_get_boot_ns)(void) = (void *) 125; + +/* + * bpf_seq_printf + * + * **bpf_seq_printf**\ () uses seq_file **seq_printf**\ () to print + * out the format string. + * The *m* represents the seq_file. The *fmt* and *fmt_size* are for + * the format string itself. The *data* and *data_len* are format string + * arguments. The *data* are a **u64** array and corresponding format string + * values are stored in the array. For strings and pointers where pointees + * are accessed, only the pointer values are stored in the *data* array. + * The *data_len* is the size of *data* in bytes. + * + * Formats **%s**, **%p{i,I}{4,6}** requires to read kernel memory. + * Reading kernel memory may fail due to either invalid address or + * valid address but requiring a major memory fault. If reading kernel memory + * fails, the string for **%s** will be an empty string, and the ip + * address for **%p{i,I}{4,6}** will be 0. Not returning error to + * bpf program is consistent with what **bpf_trace_printk**\ () does for now. + * + * Returns + * 0 on success, or a negative error in case of failure: + * + * **-EBUSY** if per-CPU memory copy buffer is busy, can try again + * by returning 1 from bpf program. + * + * **-EINVAL** if arguments are invalid, or if *fmt* is invalid/unsupported. + * + * **-E2BIG** if *fmt* contains too many format specifiers. + * + * **-EOVERFLOW** if an overflow happened: The same object will be tried again. + */ +static long (*bpf_seq_printf)(struct seq_file *m, const char *fmt, __u32 fmt_size, const void *data, __u32 data_len) = (void *) 126; + +/* + * bpf_seq_write + * + * **bpf_seq_write**\ () uses seq_file **seq_write**\ () to write the data. + * The *m* represents the seq_file. The *data* and *len* represent the + * data to write in bytes. + * + * Returns + * 0 on success, or a negative error in case of failure: + * + * **-EOVERFLOW** if an overflow happened: The same object will be tried again. + */ +static long (*bpf_seq_write)(struct seq_file *m, const void *data, __u32 len) = (void *) 127; + +/* + * bpf_sk_cgroup_id + * + * Return the cgroup v2 id of the socket *sk*. + * + * *sk* must be a non-**NULL** pointer to a full socket, e.g. one + * returned from **bpf_sk_lookup_xxx**\ (), + * **bpf_sk_fullsock**\ (), etc. The format of returned id is + * same as in **bpf_skb_cgroup_id**\ (). + * + * This helper is available only if the kernel was compiled with + * the **CONFIG_SOCK_CGROUP_DATA** configuration option. + * + * Returns + * The id is returned or 0 in case the id could not be retrieved. + */ +static __u64 (*bpf_sk_cgroup_id)(struct bpf_sock *sk) = (void *) 128; + +/* + * bpf_sk_ancestor_cgroup_id + * + * Return id of cgroup v2 that is ancestor of cgroup associated + * with the *sk* at the *ancestor_level*. The root cgroup is at + * *ancestor_level* zero and each step down the hierarchy + * increments the level. If *ancestor_level* == level of cgroup + * associated with *sk*, then return value will be same as that + * of **bpf_sk_cgroup_id**\ (). + * + * The helper is useful to implement policies based on cgroups + * that are upper in hierarchy than immediate cgroup associated + * with *sk*. + * + * The format of returned id and helper limitations are same as in + * **bpf_sk_cgroup_id**\ (). + * + * Returns + * The id is returned or 0 in case the id could not be retrieved. + */ +static __u64 (*bpf_sk_ancestor_cgroup_id)(struct bpf_sock *sk, int ancestor_level) = (void *) 129; + +/* + * bpf_ringbuf_output + * + * Copy *size* bytes from *data* into a ring buffer *ringbuf*. + * If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification + * of new data availability is sent. + * If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification + * of new data availability is sent unconditionally. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (*bpf_ringbuf_output)(void *ringbuf, void *data, __u64 size, __u64 flags) = (void *) 130; + +/* + * bpf_ringbuf_reserve + * + * Reserve *size* bytes of payload in a ring buffer *ringbuf*. + * + * Returns + * Valid pointer with *size* bytes of memory available; NULL, + * otherwise. + */ +static void *(*bpf_ringbuf_reserve)(void *ringbuf, __u64 size, __u64 flags) = (void *) 131; + +/* + * bpf_ringbuf_submit + * + * Submit reserved ring buffer sample, pointed to by *data*. + * If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification + * of new data availability is sent. + * If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification + * of new data availability is sent unconditionally. + * + * Returns + * Nothing. Always succeeds. + */ +static void (*bpf_ringbuf_submit)(void *data, __u64 flags) = (void *) 132; + +/* + * bpf_ringbuf_discard + * + * Discard reserved ring buffer sample, pointed to by *data*. + * If **BPF_RB_NO_WAKEUP** is specified in *flags*, no notification + * of new data availability is sent. + * If **BPF_RB_FORCE_WAKEUP** is specified in *flags*, notification + * of new data availability is sent unconditionally. + * + * Returns + * Nothing. Always succeeds. + */ +static void (*bpf_ringbuf_discard)(void *data, __u64 flags) = (void *) 133; + +/* + * bpf_ringbuf_query + * + * Query various characteristics of provided ring buffer. What + * exactly is queries is determined by *flags*: + * + * * **BPF_RB_AVAIL_DATA**: Amount of data not yet consumed. + * * **BPF_RB_RING_SIZE**: The size of ring buffer. + * * **BPF_RB_CONS_POS**: Consumer position (can wrap around). + * * **BPF_RB_PROD_POS**: Producer(s) position (can wrap around). + * + * Data returned is just a momentary snapshot of actual values + * and could be inaccurate, so this facility should be used to + * power heuristics and for reporting, not to make 100% correct + * calculation. + * + * Returns + * Requested value, or 0, if *flags* are not recognized. + */ +static __u64 (*bpf_ringbuf_query)(void *ringbuf, __u64 flags) = (void *) 134; + +/* + * bpf_csum_level + * + * Change the skbs checksum level by one layer up or down, or + * reset it entirely to none in order to have the stack perform + * checksum validation. The level is applicable to the following + * protocols: TCP, UDP, GRE, SCTP, FCOE. For example, a decap of + * | ETH | IP | UDP | GUE | IP | TCP | into | ETH | IP | TCP | + * through **bpf_skb_adjust_room**\ () helper with passing in + * **BPF_F_ADJ_ROOM_NO_CSUM_RESET** flag would require one call + * to **bpf_csum_level**\ () with **BPF_CSUM_LEVEL_DEC** since + * the UDP header is removed. Similarly, an encap of the latter + * into the former could be accompanied by a helper call to + * **bpf_csum_level**\ () with **BPF_CSUM_LEVEL_INC** if the + * skb is still intended to be processed in higher layers of the + * stack instead of just egressing at tc. + * + * There are three supported level settings at this time: + * + * * **BPF_CSUM_LEVEL_INC**: Increases skb->csum_level for skbs + * with CHECKSUM_UNNECESSARY. + * * **BPF_CSUM_LEVEL_DEC**: Decreases skb->csum_level for skbs + * with CHECKSUM_UNNECESSARY. + * * **BPF_CSUM_LEVEL_RESET**: Resets skb->csum_level to 0 and + * sets CHECKSUM_NONE to force checksum validation by the stack. + * * **BPF_CSUM_LEVEL_QUERY**: No-op, returns the current + * skb->csum_level. + * + * Returns + * 0 on success, or a negative error in case of failure. In the + * case of **BPF_CSUM_LEVEL_QUERY**, the current skb->csum_level + * is returned or the error code -EACCES in case the skb is not + * subject to CHECKSUM_UNNECESSARY. + */ +static long (*bpf_csum_level)(struct __sk_buff *skb, __u64 level) = (void *) 135; + +/* + * bpf_skc_to_tcp6_sock + * + * Dynamically cast a *sk* pointer to a *tcp6_sock* pointer. + * + * Returns + * *sk* if casting is valid, or NULL otherwise. + */ +static struct tcp6_sock *(*bpf_skc_to_tcp6_sock)(void *sk) = (void *) 136; + +/* + * bpf_skc_to_tcp_sock + * + * Dynamically cast a *sk* pointer to a *tcp_sock* pointer. + * + * Returns + * *sk* if casting is valid, or NULL otherwise. + */ +static struct tcp_sock *(*bpf_skc_to_tcp_sock)(void *sk) = (void *) 137; + +/* + * bpf_skc_to_tcp_timewait_sock + * + * Dynamically cast a *sk* pointer to a *tcp_timewait_sock* pointer. + * + * Returns + * *sk* if casting is valid, or NULL otherwise. + */ +static struct tcp_timewait_sock *(*bpf_skc_to_tcp_timewait_sock)(void *sk) = (void *) 138; + +/* + * bpf_skc_to_tcp_request_sock + * + * Dynamically cast a *sk* pointer to a *tcp_request_sock* pointer. + * + * Returns + * *sk* if casting is valid, or NULL otherwise. + */ +static struct tcp_request_sock *(*bpf_skc_to_tcp_request_sock)(void *sk) = (void *) 139; + +/* + * bpf_skc_to_udp6_sock + * + * Dynamically cast a *sk* pointer to a *udp6_sock* pointer. + * + * Returns + * *sk* if casting is valid, or NULL otherwise. + */ +static struct udp6_sock *(*bpf_skc_to_udp6_sock)(void *sk) = (void *) 140; + +/* + * bpf_get_task_stack + * + * Return a user or a kernel stack in bpf program provided buffer. + * To achieve this, the helper needs *task*, which is a valid + * pointer to struct task_struct. To store the stacktrace, the + * bpf program provides *buf* with a nonnegative *size*. + * + * The last argument, *flags*, holds the number of stack frames to + * skip (from 0 to 255), masked with + * **BPF_F_SKIP_FIELD_MASK**. The next bits can be used to set + * the following flags: + * + * **BPF_F_USER_STACK** + * Collect a user space stack instead of a kernel stack. + * **BPF_F_USER_BUILD_ID** + * Collect buildid+offset instead of ips for user stack, + * only valid if **BPF_F_USER_STACK** is also specified. + * + * **bpf_get_task_stack**\ () can collect up to + * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject + * to sufficient large buffer size. Note that + * this limit can be controlled with the **sysctl** program, and + * that it should be manually increased in order to profile long + * user stacks (such as stacks for Java programs). To do so, use: + * + * :: + * + * # sysctl kernel.perf_event_max_stack= + * + * Returns + * A non-negative value equal to or less than *size* on success, + * or a negative error in case of failure. + */ +static long (*bpf_get_task_stack)(struct task_struct *task, void *buf, __u32 size, __u64 flags) = (void *) 141; + + diff --git a/vendor/github.com/cilium/ebpf/examples/headers/bpf_helpers.h b/vendor/github.com/cilium/ebpf/examples/headers/bpf_helpers.h new file mode 100644 index 0000000000000..bc14db706b88a --- /dev/null +++ b/vendor/github.com/cilium/ebpf/examples/headers/bpf_helpers.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __BPF_HELPERS__ +#define __BPF_HELPERS__ + +/* + * Note that bpf programs need to include either + * vmlinux.h (auto-generated from BTF) or linux/types.h + * in advance since bpf_helper_defs.h uses such types + * as __u64. + */ +#include "bpf_helper_defs.h" + +#define __uint(name, val) int (*name)[val] +#define __type(name, val) typeof(val) *name +#define __array(name, val) typeof(val) *name[] + +/* Helper macro to print out debug messages */ +#define bpf_printk(fmt, ...) \ +({ \ + char ____fmt[] = fmt; \ + bpf_trace_printk(____fmt, sizeof(____fmt), \ + ##__VA_ARGS__); \ +}) + +/* + * Helper macro to place programs, maps, license in + * different sections in elf_bpf file. Section names + * are interpreted by elf_bpf loader + */ +#define SEC(NAME) __attribute__((section(NAME), used)) + +#ifndef __always_inline +#define __always_inline __attribute__((always_inline)) +#endif +#ifndef __weak +#define __weak __attribute__((weak)) +#endif + +/* + * Helper macro to manipulate data structures + */ +#ifndef offsetof +#define offsetof(TYPE, MEMBER) __builtin_offsetof(TYPE, MEMBER) +#endif +#ifndef container_of +#define container_of(ptr, type, member) \ + ({ \ + void *__mptr = (void *)(ptr); \ + ((type *)(__mptr - offsetof(type, member))); \ + }) +#endif + +/* + * Helper structure used by eBPF C program + * to describe BPF map attributes to libbpf loader + */ +struct bpf_map_def { + unsigned int type; + unsigned int key_size; + unsigned int value_size; + unsigned int max_entries; + unsigned int map_flags; +}; + +enum libbpf_pin_type { + LIBBPF_PIN_NONE, + /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ + LIBBPF_PIN_BY_NAME, +}; + +enum libbpf_tristate { + TRI_NO = 0, + TRI_YES = 1, + TRI_MODULE = 2, +}; + +#define __kconfig __attribute__((section(".kconfig"))) +#define __ksym __attribute__((section(".ksyms"))) + +#endif diff --git a/vendor/github.com/cilium/ebpf/examples/headers/common.h b/vendor/github.com/cilium/ebpf/examples/headers/common.h new file mode 100644 index 0000000000000..ecb3f575e984b --- /dev/null +++ b/vendor/github.com/cilium/ebpf/examples/headers/common.h @@ -0,0 +1,65 @@ +// This is a compact version of `vmlinux.h` to be used in the examples using C code. + +#ifndef __VMLINUX_H__ +#define __VMLINUX_H__ + +typedef unsigned char __u8; +typedef short int __s16; +typedef short unsigned int __u16; +typedef int __s32; +typedef unsigned int __u32; +typedef long long int __s64; +typedef long long unsigned int __u64; +typedef __u8 u8; +typedef __s16 s16; +typedef __u16 u16; +typedef __s32 s32; +typedef __u32 u32; +typedef __s64 s64; +typedef __u64 u64; +typedef __u16 __le16; +typedef __u16 __be16; +typedef __u32 __be32; +typedef __u64 __be64; +typedef __u32 __wsum; + +enum bpf_map_type { + BPF_MAP_TYPE_UNSPEC = 0, + BPF_MAP_TYPE_HASH = 1, + BPF_MAP_TYPE_ARRAY = 2, + BPF_MAP_TYPE_PROG_ARRAY = 3, + BPF_MAP_TYPE_PERF_EVENT_ARRAY = 4, + BPF_MAP_TYPE_PERCPU_HASH = 5, + BPF_MAP_TYPE_PERCPU_ARRAY = 6, + BPF_MAP_TYPE_STACK_TRACE = 7, + BPF_MAP_TYPE_CGROUP_ARRAY = 8, + BPF_MAP_TYPE_LRU_HASH = 9, + BPF_MAP_TYPE_LRU_PERCPU_HASH = 10, + BPF_MAP_TYPE_LPM_TRIE = 11, + BPF_MAP_TYPE_ARRAY_OF_MAPS = 12, + BPF_MAP_TYPE_HASH_OF_MAPS = 13, + BPF_MAP_TYPE_DEVMAP = 14, + BPF_MAP_TYPE_SOCKMAP = 15, + BPF_MAP_TYPE_CPUMAP = 16, + BPF_MAP_TYPE_XSKMAP = 17, + BPF_MAP_TYPE_SOCKHASH = 18, + BPF_MAP_TYPE_CGROUP_STORAGE = 19, + BPF_MAP_TYPE_REUSEPORT_SOCKARRAY = 20, + BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE = 21, + BPF_MAP_TYPE_QUEUE = 22, + BPF_MAP_TYPE_STACK = 23, + BPF_MAP_TYPE_SK_STORAGE = 24, + BPF_MAP_TYPE_DEVMAP_HASH = 25, + BPF_MAP_TYPE_STRUCT_OPS = 26, + BPF_MAP_TYPE_RINGBUF = 27, + BPF_MAP_TYPE_INODE_STORAGE = 28, +}; + +enum { + BPF_ANY = 0, + BPF_NOEXIST = 1, + BPF_EXIST = 2, + BPF_F_LOCK = 4, +}; + +#endif /* __VMLINUX_H__ */ diff --git a/vendor/github.com/cilium/ebpf/examples/kprobe/bpf/kprobe_example.c b/vendor/github.com/cilium/ebpf/examples/kprobe/bpf/kprobe_example.c new file mode 100644 index 0000000000000..411c4f9308edc --- /dev/null +++ b/vendor/github.com/cilium/ebpf/examples/kprobe/bpf/kprobe_example.c @@ -0,0 +1,26 @@ +#include "common.h" +#include "bpf_helpers.h" + +char __license[] SEC("license") = "Dual MIT/GPL"; + +struct bpf_map_def SEC("maps") kprobe_map = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(u32), + .value_size = sizeof(u64), + .max_entries = 1, +}; + +SEC("kprobe/__x64_sys_execve") +int kprobe_execve() { + u32 key = 0; + u64 initval = 1, *valp; + + valp = bpf_map_lookup_elem(&kprobe_map, &key); + if (!valp) { + bpf_map_update_elem(&kprobe_map, &key, &initval, BPF_ANY); + return 0; + } + __sync_fetch_and_add(valp, 1); + + return 0; +} diff --git a/vendor/github.com/cilium/ebpf/go.mod b/vendor/github.com/cilium/ebpf/go.mod index a05cf85eddd36..df8139621c3e5 100644 --- a/vendor/github.com/cilium/ebpf/go.mod +++ b/vendor/github.com/cilium/ebpf/go.mod @@ -1,5 +1,9 @@ module github.com/cilium/ebpf -go 1.13 +go 1.15 -require golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 +require ( + github.com/frankban/quicktest v1.11.3 + github.com/google/go-cmp v0.5.4 + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c +) diff --git a/vendor/github.com/cilium/ebpf/info.go b/vendor/github.com/cilium/ebpf/info.go new file mode 100644 index 0000000000000..b95131ef57262 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/info.go @@ -0,0 +1,239 @@ +package ebpf + +import ( + "bufio" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "strings" + "syscall" + "time" + + "github.com/cilium/ebpf/internal" +) + +// MapInfo describes a map. +type MapInfo struct { + Type MapType + id MapID + KeySize uint32 + ValueSize uint32 + MaxEntries uint32 + Flags uint32 + // Name as supplied by user space at load time. + Name string +} + +func newMapInfoFromFd(fd *internal.FD) (*MapInfo, error) { + info, err := bpfGetMapInfoByFD(fd) + if errors.Is(err, syscall.EINVAL) { + return newMapInfoFromProc(fd) + } + if err != nil { + return nil, err + } + + return &MapInfo{ + MapType(info.map_type), + MapID(info.id), + info.key_size, + info.value_size, + info.max_entries, + info.map_flags, + // name is available from 4.15. + internal.CString(info.name[:]), + }, nil +} + +func newMapInfoFromProc(fd *internal.FD) (*MapInfo, error) { + var mi MapInfo + err := scanFdInfo(fd, map[string]interface{}{ + "map_type": &mi.Type, + "key_size": &mi.KeySize, + "value_size": &mi.ValueSize, + "max_entries": &mi.MaxEntries, + "map_flags": &mi.Flags, + }) + if err != nil { + return nil, err + } + return &mi, nil +} + +// ID returns the map ID. +// +// Available from 4.13. +// +// The bool return value indicates whether this optional field is available. +func (mi *MapInfo) ID() (MapID, bool) { + return mi.id, mi.id > 0 +} + +// programStats holds statistics of a program. +type programStats struct { + // Total accumulated runtime of the program ins ns. + runtime time.Duration + // Total number of times the program was called. + runCount uint64 +} + +// ProgramInfo describes a program. +type ProgramInfo struct { + Type ProgramType + id ProgramID + // Truncated hash of the BPF bytecode. + Tag string + // Name as supplied by user space at load time. + Name string + + stats *programStats +} + +func newProgramInfoFromFd(fd *internal.FD) (*ProgramInfo, error) { + info, err := bpfGetProgInfoByFD(fd) + if errors.Is(err, syscall.EINVAL) { + return newProgramInfoFromProc(fd) + } + if err != nil { + return nil, err + } + + return &ProgramInfo{ + Type: ProgramType(info.prog_type), + id: ProgramID(info.id), + // tag is available if the kernel supports BPF_PROG_GET_INFO_BY_FD. + Tag: hex.EncodeToString(info.tag[:]), + // name is available from 4.15. + Name: internal.CString(info.name[:]), + stats: &programStats{ + runtime: time.Duration(info.run_time_ns), + runCount: info.run_cnt, + }, + }, nil +} + +func newProgramInfoFromProc(fd *internal.FD) (*ProgramInfo, error) { + var info ProgramInfo + err := scanFdInfo(fd, map[string]interface{}{ + "prog_type": &info.Type, + "prog_tag": &info.Tag, + }) + if errors.Is(err, errMissingFields) { + return nil, &internal.UnsupportedFeatureError{ + Name: "reading program info from /proc/self/fdinfo", + MinimumVersion: internal.Version{4, 10, 0}, + } + } + if err != nil { + return nil, err + } + + return &info, nil +} + +// ID returns the program ID. +// +// Available from 4.13. +// +// The bool return value indicates whether this optional field is available. +func (pi *ProgramInfo) ID() (ProgramID, bool) { + return pi.id, pi.id > 0 +} + +// RunCount returns the total number of times the program was called. +// +// Can return 0 if the collection of statistics is not enabled. See EnableStats(). +// The bool return value indicates whether this optional field is available. +func (pi *ProgramInfo) RunCount() (uint64, bool) { + if pi.stats != nil { + return pi.stats.runCount, true + } + return 0, false +} + +// Runtime returns the total accumulated runtime of the program. +// +// Can return 0 if the collection of statistics is not enabled. See EnableStats(). +// The bool return value indicates whether this optional field is available. +func (pi *ProgramInfo) Runtime() (time.Duration, bool) { + if pi.stats != nil { + return pi.stats.runtime, true + } + return time.Duration(0), false +} + +func scanFdInfo(fd *internal.FD, fields map[string]interface{}) error { + raw, err := fd.Value() + if err != nil { + return err + } + + fh, err := os.Open(fmt.Sprintf("/proc/self/fdinfo/%d", raw)) + if err != nil { + return err + } + defer fh.Close() + + if err := scanFdInfoReader(fh, fields); err != nil { + return fmt.Errorf("%s: %w", fh.Name(), err) + } + return nil +} + +var errMissingFields = errors.New("missing fields") + +func scanFdInfoReader(r io.Reader, fields map[string]interface{}) error { + var ( + scanner = bufio.NewScanner(r) + scanned int + ) + + for scanner.Scan() { + parts := strings.SplitN(scanner.Text(), "\t", 2) + if len(parts) != 2 { + continue + } + + name := strings.TrimSuffix(parts[0], ":") + field, ok := fields[string(name)] + if !ok { + continue + } + + if n, err := fmt.Sscanln(parts[1], field); err != nil || n != 1 { + return fmt.Errorf("can't parse field %s: %v", name, err) + } + + scanned++ + } + + if err := scanner.Err(); err != nil { + return err + } + + if scanned != len(fields) { + return errMissingFields + } + + return nil +} + +// EnableStats starts the measuring of the runtime +// and run counts of eBPF programs. +// +// Collecting statistics can have an impact on the performance. +// +// Requires at least 5.8. +func EnableStats(which uint32) (io.Closer, error) { + attr := internal.BPFEnableStatsAttr{ + StatsType: which, + } + + fd, err := internal.BPFEnableStats(&attr) + if err != nil { + return nil, err + } + return fd, nil +} diff --git a/vendor/github.com/cilium/ebpf/internal/btf/btf.go b/vendor/github.com/cilium/ebpf/internal/btf/btf.go index 3dd000a284e7d..1e66d94765a13 100644 --- a/vendor/github.com/cilium/ebpf/internal/btf/btf.go +++ b/vendor/github.com/cilium/ebpf/internal/btf/btf.go @@ -29,12 +29,14 @@ var ( // Spec represents decoded BTF. type Spec struct { - rawTypes []rawType - strings stringTable - types map[string][]Type - funcInfos map[string]extInfo - lineInfos map[string]extInfo - byteOrder binary.ByteOrder + rawTypes []rawType + strings stringTable + types []Type + namedTypes map[string][]namedType + funcInfos map[string]extInfo + lineInfos map[string]extInfo + coreRelos map[string]bpfCoreRelos + byteOrder binary.ByteOrder } type btfHeader struct { @@ -53,35 +55,15 @@ type btfHeader struct { // // Returns a nil Spec and no error if no BTF was present. func LoadSpecFromReader(rd io.ReaderAt) (*Spec, error) { - file, err := elf.NewFile(rd) + file, err := internal.NewSafeELFFile(rd) if err != nil { return nil, err } defer file.Close() - var ( - btfSection *elf.Section - btfExtSection *elf.Section - sectionSizes = make(map[string]uint32) - ) - - for _, sec := range file.Sections { - switch sec.Name { - case ".BTF": - btfSection = sec - case ".BTF.ext": - btfExtSection = sec - default: - if sec.Type != elf.SHT_PROGBITS && sec.Type != elf.SHT_NOBITS { - break - } - - if sec.Size > math.MaxUint32 { - return nil, fmt.Errorf("section %s exceeds maximum size", sec.Name) - } - - sectionSizes[sec.Name] = uint32(sec.Size) - } + btfSection, btfExtSection, sectionSizes, err := findBtfSections(file) + if err != nil { + return nil, err } if btfSection == nil { @@ -100,6 +82,10 @@ func LoadSpecFromReader(rd io.ReaderAt) (*Spec, error) { continue } + if int(symbol.Section) >= len(file.Sections) { + return nil, fmt.Errorf("symbol %s: invalid section %d", symbol.Name, symbol.Section) + } + secName := file.Sections[symbol.Section].Name if _, ok := sectionSizes[secName]; !ok { continue @@ -121,7 +107,7 @@ func LoadSpecFromReader(rd io.ReaderAt) (*Spec, error) { return spec, nil } - spec.funcInfos, spec.lineInfos, err = parseExtInfos(btfExtSection.Open(), file.ByteOrder, spec.strings) + spec.funcInfos, spec.lineInfos, spec.coreRelos, err = parseExtInfos(btfExtSection.Open(), file.ByteOrder, spec.strings) if err != nil { return nil, fmt.Errorf("can't read ext info: %w", err) } @@ -129,6 +115,51 @@ func LoadSpecFromReader(rd io.ReaderAt) (*Spec, error) { return spec, nil } +func findBtfSections(file *internal.SafeELFFile) (*elf.Section, *elf.Section, map[string]uint32, error) { + var ( + btfSection *elf.Section + btfExtSection *elf.Section + sectionSizes = make(map[string]uint32) + ) + + for _, sec := range file.Sections { + switch sec.Name { + case ".BTF": + btfSection = sec + case ".BTF.ext": + btfExtSection = sec + default: + if sec.Type != elf.SHT_PROGBITS && sec.Type != elf.SHT_NOBITS { + break + } + + if sec.Size > math.MaxUint32 { + return nil, nil, nil, fmt.Errorf("section %s exceeds maximum size", sec.Name) + } + + sectionSizes[sec.Name] = uint32(sec.Size) + } + } + return btfSection, btfExtSection, sectionSizes, nil +} + +func loadSpecFromVmlinux(rd io.ReaderAt) (*Spec, error) { + file, err := internal.NewSafeELFFile(rd) + if err != nil { + return nil, err + } + defer file.Close() + + btfSection, _, _, err := findBtfSections(file) + if err != nil { + return nil, fmt.Errorf(".BTF ELF section: %s", err) + } + if btfSection == nil { + return nil, fmt.Errorf("unable to find .BTF ELF section") + } + return loadNakedSpec(btfSection.Open(), file.ByteOrder, nil, nil) +} + func loadNakedSpec(btf io.ReadSeeker, bo binary.ByteOrder, sectionSizes map[string]uint32, variableOffsets map[variable]uint32) (*Spec, error) { rawTypes, rawStrings, err := parseBTF(btf, bo) if err != nil { @@ -140,16 +171,17 @@ func loadNakedSpec(btf io.ReadSeeker, bo binary.ByteOrder, sectionSizes map[stri return nil, err } - types, err := inflateRawTypes(rawTypes, rawStrings) + types, typesByName, err := inflateRawTypes(rawTypes, rawStrings) if err != nil { return nil, err } return &Spec{ - rawTypes: rawTypes, - types: types, - strings: rawStrings, - byteOrder: bo, + rawTypes: rawTypes, + namedTypes: typesByName, + types: types, + strings: rawStrings, + byteOrder: bo, }, nil } @@ -176,16 +208,43 @@ func LoadKernelSpec() (*Spec, error) { } func loadKernelSpec() (*Spec, error) { + release, err := unix.KernelRelease() + if err != nil { + return nil, fmt.Errorf("can't read kernel release number: %w", err) + } + fh, err := os.Open("/sys/kernel/btf/vmlinux") - if os.IsNotExist(err) { - return nil, fmt.Errorf("can't open kernel BTF at /sys/kernel/btf/vmlinux: %w", ErrNotFound) + if err == nil { + defer fh.Close() + + return loadNakedSpec(fh, internal.NativeEndian, nil, nil) } - if err != nil { - return nil, fmt.Errorf("can't read kernel BTF: %s", err) + + // use same list of locations as libbpf + // https://github.com/libbpf/libbpf/blob/9a3a42608dbe3731256a5682a125ac1e23bced8f/src/btf.c#L3114-L3122 + locations := []string{ + "/boot/vmlinux-%s", + "/lib/modules/%s/vmlinux-%[1]s", + "/lib/modules/%s/build/vmlinux", + "/usr/lib/modules/%s/kernel/vmlinux", + "/usr/lib/debug/boot/vmlinux-%s", + "/usr/lib/debug/boot/vmlinux-%s.debug", + "/usr/lib/debug/lib/modules/%s/vmlinux", } - defer fh.Close() - return loadNakedSpec(fh, internal.NativeEndian, nil, nil) + for _, loc := range locations { + path := fmt.Sprintf(loc, release) + + fh, err := os.Open(path) + if err != nil { + continue + } + defer fh.Close() + + return loadSpecFromVmlinux(fh) + } + + return nil, fmt.Errorf("no BTF for kernel version %s: %w", release, internal.ErrNotSupported) } func parseBTF(btf io.ReadSeeker, bo binary.ByteOrder) ([]rawType, stringTable, error) { @@ -259,10 +318,14 @@ func fixupDatasec(rawTypes []rawType, rawStrings stringTable, sectionSizes map[s return err } - if name == ".kconfig" || name == ".ksym" { + if name == ".kconfig" || name == ".ksyms" { return fmt.Errorf("reference to %s: %w", name, ErrNotSupported) } + if rawTypes[i].SizeType != 0 { + continue + } + size, ok := sectionSizes[name] if !ok { return fmt.Errorf("data section %s: missing size", name) @@ -369,54 +432,19 @@ func (s *Spec) Program(name string, length uint64) (*Program, error) { return nil, errors.New("length musn't be zero") } - if s.funcInfos == nil && s.lineInfos == nil { + if s.funcInfos == nil && s.lineInfos == nil && s.coreRelos == nil { return nil, fmt.Errorf("BTF for section %s: %w", name, ErrNoExtendedInfo) } funcInfos, funcOK := s.funcInfos[name] lineInfos, lineOK := s.lineInfos[name] + coreRelos, coreOK := s.coreRelos[name] - if !funcOK && !lineOK { + if !funcOK && !lineOK && !coreOK { return nil, fmt.Errorf("no extended BTF info for section %s", name) } - return &Program{s, length, funcInfos, lineInfos}, nil -} - -// Map finds the BTF for a map. -// -// Returns an error if there is no BTF for the given name. -func (s *Spec) Map(name string) (*Map, []Member, error) { - var mapVar Var - if err := s.FindType(name, &mapVar); err != nil { - return nil, nil, err - } - - mapStruct, ok := mapVar.Type.(*Struct) - if !ok { - return nil, nil, fmt.Errorf("expected struct, have %s", mapVar.Type) - } - - var key, value Type - for _, member := range mapStruct.Members { - switch member.Name { - case "key": - key = member.Type - - case "value": - value = member.Type - } - } - - if key == nil { - key = (*Void)(nil) - } - - if value == nil { - value = (*Void)(nil) - } - - return &Map{s, key, value}, mapStruct.Members, nil + return &Program{s, length, funcInfos, lineInfos, coreRelos}, nil } // Datasec returns the BTF required to create maps which represent data sections. @@ -426,7 +454,8 @@ func (s *Spec) Datasec(name string) (*Map, error) { return nil, fmt.Errorf("data section %s: can't get BTF: %w", name, err) } - return &Map{s, &Void{}, &datasec}, nil + m := NewMap(s, &Void{}, &datasec) + return &m, nil } // FindType searches for a type with a specific name. @@ -441,11 +470,16 @@ func (s *Spec) FindType(name string, typ Type) error { candidate Type ) - for _, typ := range s.types[name] { + for _, typ := range s.namedTypes[essentialName(name)] { if reflect.TypeOf(typ) != wanted { continue } + // Match against the full name, not just the essential one. + if typ.name() != name { + continue + } + if candidate != nil { return fmt.Errorf("type %s: multiple candidates for %T", name, typ) } @@ -532,6 +566,23 @@ type Map struct { key, value Type } +// NewMap returns a new Map containing the given values. +// The key and value arguments are initialized to Void if nil values are given. +func NewMap(spec *Spec, key Type, value Type) Map { + if key == nil { + key = &Void{} + } + if value == nil { + value = &Void{} + } + + return Map{ + spec: spec, + key: key, + value: value, + } +} + // MapSpec should be a method on Map, but is a free function // to hide it from users of the ebpf package. func MapSpec(m *Map) *Spec { @@ -555,6 +606,7 @@ type Program struct { spec *Spec length uint64 funcInfos, lineInfos extInfo + coreRelos bpfCoreRelos } // ProgramSpec returns the Spec needed for loading function and line infos into the kernel. @@ -580,9 +632,10 @@ func ProgramAppend(s, other *Program) error { return fmt.Errorf("line infos: %w", err) } - s.length += other.length s.funcInfos = funcInfos s.lineInfos = lineInfos + s.coreRelos = s.coreRelos.append(other.coreRelos, s.length) + s.length += other.length return nil } @@ -612,6 +665,19 @@ func ProgramLineInfos(s *Program) (recordSize uint32, bytes []byte, err error) { return s.lineInfos.recordSize, bytes, nil } +// ProgramRelocations returns the CO-RE relocations required to adjust the +// program to the target. +// +// This is a free function instead of a method to hide it from users +// of package ebpf. +func ProgramRelocations(s *Program, target *Spec) (map[uint64]Relocation, error) { + if len(s.coreRelos) == 0 { + return nil, nil + } + + return coreRelocate(s.spec, target, s.coreRelos) +} + type bpfLoadBTFAttr struct { btf internal.Pointer logBuf internal.Pointer @@ -621,9 +687,7 @@ type bpfLoadBTFAttr struct { } func bpfLoadBTF(attr *bpfLoadBTFAttr) (*internal.FD, error) { - const _BTFLoad = 18 - - fd, err := internal.BPF(_BTFLoad, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + fd, err := internal.BPF(internal.BPF_BTF_LOAD, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) if err != nil { return nil, err } @@ -653,7 +717,7 @@ func marshalBTF(types interface{}, strings []byte, bo binary.ByteOrder) []byte { return buf.Bytes() } -var haveBTF = internal.FeatureTest("BTF", "5.1", func() (bool, error) { +var haveBTF = internal.FeatureTest("BTF", "5.1", func() error { var ( types struct { Integer btfType @@ -677,15 +741,24 @@ var haveBTF = internal.FeatureTest("BTF", "5.1", func() (bool, error) { btf: internal.NewSlicePointer(btf), btfSize: uint32(len(btf)), }) - if err == nil { - fd.Close() + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) { + // Treat both EINVAL and EPERM as not supported: loading the program + // might still succeed without BTF. + return internal.ErrNotSupported + } + if err != nil { + return err } - // Check for EINVAL specifically, rather than err != nil since we - // otherwise misdetect due to insufficient permissions. - return !errors.Is(err, unix.EINVAL), nil + + fd.Close() + return nil }) -var haveFuncLinkage = internal.FeatureTest("BTF func linkage", "5.6", func() (bool, error) { +var haveFuncLinkage = internal.FeatureTest("BTF func linkage", "5.6", func() error { + if err := haveBTF(); err != nil { + return err + } + var ( types struct { FuncProto btfType @@ -706,11 +779,13 @@ var haveFuncLinkage = internal.FeatureTest("BTF func linkage", "5.6", func() (bo btf: internal.NewSlicePointer(btf), btfSize: uint32(len(btf)), }) - if err == nil { - fd.Close() + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported + } + if err != nil { + return err } - // Check for EINVAL specifically, rather than err != nil since we - // otherwise misdetect due to insufficient permissions. - return !errors.Is(err, unix.EINVAL), nil + fd.Close() + return nil }) diff --git a/vendor/github.com/cilium/ebpf/internal/btf/btf_types.go b/vendor/github.com/cilium/ebpf/internal/btf/btf_types.go index e34a87ecb457d..a4cde3fe82706 100644 --- a/vendor/github.com/cilium/ebpf/internal/btf/btf_types.go +++ b/vendor/github.com/cilium/ebpf/internal/btf/btf_types.go @@ -40,10 +40,12 @@ const ( ) const ( - btfTypeKindShift = 24 - btfTypeKindLen = 4 - btfTypeVlenShift = 0 - btfTypeVlenMask = 16 + btfTypeKindShift = 24 + btfTypeKindLen = 4 + btfTypeVlenShift = 0 + btfTypeVlenMask = 16 + btfTypeKindFlagShift = 31 + btfTypeKindFlagMask = 1 ) // btfType is equivalent to struct btf_type in Documentation/bpf/btf.rst. @@ -136,6 +138,10 @@ func (bt *btfType) SetVlen(vlen int) { bt.setInfo(uint32(vlen), btfTypeVlenMask, btfTypeVlenShift) } +func (bt *btfType) KindFlag() bool { + return bt.info(btfTypeKindFlagMask, btfTypeKindFlagShift) == 1 +} + func (bt *btfType) Linkage() btfFuncLinkage { return btfFuncLinkage(bt.info(btfTypeVlenMask, btfTypeVlenShift)) } @@ -257,3 +263,7 @@ func readTypes(r io.Reader, bo binary.ByteOrder) ([]rawType, error) { types = append(types, rawType{header, data}) } } + +func intEncoding(raw uint32) (IntEncoding, uint32, byte) { + return IntEncoding((raw & 0x0f000000) >> 24), (raw & 0x00ff0000) >> 16, byte(raw & 0x000000ff) +} diff --git a/vendor/github.com/cilium/ebpf/internal/btf/core.go b/vendor/github.com/cilium/ebpf/internal/btf/core.go new file mode 100644 index 0000000000000..52b59ed189f48 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/internal/btf/core.go @@ -0,0 +1,388 @@ +package btf + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" +) + +// Code in this file is derived from libbpf, which is available under a BSD +// 2-Clause license. + +// Relocation describes a CO-RE relocation. +type Relocation struct { + Current uint32 + New uint32 +} + +func (r Relocation) equal(other Relocation) bool { + return r.Current == other.Current && r.New == other.New +} + +// coreReloKind is the type of CO-RE relocation +type coreReloKind uint32 + +const ( + reloFieldByteOffset coreReloKind = iota /* field byte offset */ + reloFieldByteSize /* field size in bytes */ + reloFieldExists /* field existence in target kernel */ + reloFieldSigned /* field signedness (0 - unsigned, 1 - signed) */ + reloFieldLShiftU64 /* bitfield-specific left bitshift */ + reloFieldRShiftU64 /* bitfield-specific right bitshift */ + reloTypeIDLocal /* type ID in local BPF object */ + reloTypeIDTarget /* type ID in target kernel */ + reloTypeExists /* type existence in target kernel */ + reloTypeSize /* type size in bytes */ + reloEnumvalExists /* enum value existence in target kernel */ + reloEnumvalValue /* enum value integer value */ +) + +func (k coreReloKind) String() string { + switch k { + case reloFieldByteOffset: + return "byte_off" + case reloFieldByteSize: + return "byte_sz" + case reloFieldExists: + return "field_exists" + case reloFieldSigned: + return "signed" + case reloFieldLShiftU64: + return "lshift_u64" + case reloFieldRShiftU64: + return "rshift_u64" + case reloTypeIDLocal: + return "local_type_id" + case reloTypeIDTarget: + return "target_type_id" + case reloTypeExists: + return "type_exists" + case reloTypeSize: + return "type_size" + case reloEnumvalExists: + return "enumval_exists" + case reloEnumvalValue: + return "enumval_value" + default: + return "unknown" + } +} + +func coreRelocate(local, target *Spec, coreRelos bpfCoreRelos) (map[uint64]Relocation, error) { + if target == nil { + var err error + target, err = loadKernelSpec() + if err != nil { + return nil, err + } + } + + if local.byteOrder != target.byteOrder { + return nil, fmt.Errorf("can't relocate %s against %s", local.byteOrder, target.byteOrder) + } + + relocations := make(map[uint64]Relocation, len(coreRelos)) + for _, relo := range coreRelos { + accessorStr, err := local.strings.Lookup(relo.AccessStrOff) + if err != nil { + return nil, err + } + + accessor, err := parseCoreAccessor(accessorStr) + if err != nil { + return nil, fmt.Errorf("accessor %q: %s", accessorStr, err) + } + + if int(relo.TypeID) >= len(local.types) { + return nil, fmt.Errorf("invalid type id %d", relo.TypeID) + } + + typ := local.types[relo.TypeID] + + if relo.ReloKind == reloTypeIDLocal { + relocations[uint64(relo.InsnOff)] = Relocation{ + uint32(typ.ID()), + uint32(typ.ID()), + } + continue + } + + named, ok := typ.(namedType) + if !ok || named.name() == "" { + return nil, fmt.Errorf("relocate anonymous type %s: %w", typ.String(), ErrNotSupported) + } + + name := essentialName(named.name()) + res, err := coreCalculateRelocation(typ, target.namedTypes[name], relo.ReloKind, accessor) + if err != nil { + return nil, fmt.Errorf("relocate %s: %w", name, err) + } + + relocations[uint64(relo.InsnOff)] = res + } + + return relocations, nil +} + +var errAmbiguousRelocation = errors.New("ambiguous relocation") + +func coreCalculateRelocation(local Type, targets []namedType, kind coreReloKind, localAccessor coreAccessor) (Relocation, error) { + var relos []Relocation + var matches []Type + for _, target := range targets { + switch kind { + case reloTypeIDTarget: + if localAccessor[0] != 0 { + return Relocation{}, fmt.Errorf("%s: unexpected non-zero accessor", kind) + } + + if compat, err := coreAreTypesCompatible(local, target); err != nil { + return Relocation{}, fmt.Errorf("%s: %s", kind, err) + } else if !compat { + continue + } + + relos = append(relos, Relocation{uint32(target.ID()), uint32(target.ID())}) + + default: + return Relocation{}, fmt.Errorf("relocation %s: %w", kind, ErrNotSupported) + } + matches = append(matches, target) + } + + if len(relos) == 0 { + // TODO: Add switch for existence checks like reloEnumvalExists here. + + // TODO: This might have to be poisoned. + return Relocation{}, fmt.Errorf("no relocation found, tried %v", targets) + } + + relo := relos[0] + for _, altRelo := range relos[1:] { + if !altRelo.equal(relo) { + return Relocation{}, fmt.Errorf("multiple types %v match: %w", matches, errAmbiguousRelocation) + } + } + + return relo, nil +} + +/* coreAccessor contains a path through a struct. It contains at least one index. + * + * The interpretation depends on the kind of the relocation. The following is + * taken from struct bpf_core_relo in libbpf_internal.h: + * + * - for field-based relocations, string encodes an accessed field using + * a sequence of field and array indices, separated by colon (:). It's + * conceptually very close to LLVM's getelementptr ([0]) instruction's + * arguments for identifying offset to a field. + * - for type-based relocations, strings is expected to be just "0"; + * - for enum value-based relocations, string contains an index of enum + * value within its enum type; + * + * Example to provide a better feel. + * + * struct sample { + * int a; + * struct { + * int b[10]; + * }; + * }; + * + * struct sample s = ...; + * int x = &s->a; // encoded as "0:0" (a is field #0) + * int y = &s->b[5]; // encoded as "0:1:0:5" (anon struct is field #1, + * // b is field #0 inside anon struct, accessing elem #5) + * int z = &s[10]->b; // encoded as "10:1" (ptr is used as an array) + */ +type coreAccessor []int + +func parseCoreAccessor(accessor string) (coreAccessor, error) { + if accessor == "" { + return nil, fmt.Errorf("empty accessor") + } + + var result coreAccessor + parts := strings.Split(accessor, ":") + for _, part := range parts { + // 31 bits to avoid overflowing int on 32 bit platforms. + index, err := strconv.ParseUint(part, 10, 31) + if err != nil { + return nil, fmt.Errorf("accessor index %q: %s", part, err) + } + + result = append(result, int(index)) + } + + return result, nil +} + +/* The comment below is from bpf_core_types_are_compat in libbpf.c: + * + * Check local and target types for compatibility. This check is used for + * type-based CO-RE relocations and follow slightly different rules than + * field-based relocations. This function assumes that root types were already + * checked for name match. Beyond that initial root-level name check, names + * are completely ignored. Compatibility rules are as follows: + * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but + * kind should match for local and target types (i.e., STRUCT is not + * compatible with UNION); + * - for ENUMs, the size is ignored; + * - for INT, size and signedness are ignored; + * - for ARRAY, dimensionality is ignored, element types are checked for + * compatibility recursively; + * - CONST/VOLATILE/RESTRICT modifiers are ignored; + * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; + * - FUNC_PROTOs are compatible if they have compatible signature: same + * number of input args and compatible return and argument types. + * These rules are not set in stone and probably will be adjusted as we get + * more experience with using BPF CO-RE relocations. + */ +func coreAreTypesCompatible(localType Type, targetType Type) (bool, error) { + var ( + localTs, targetTs typeDeque + l, t = &localType, &targetType + depth = 0 + ) + + for ; l != nil && t != nil; l, t = localTs.shift(), targetTs.shift() { + if depth >= maxTypeDepth { + return false, errors.New("types are nested too deep") + } + + localType = skipQualifierAndTypedef(*l) + targetType = skipQualifierAndTypedef(*t) + + if reflect.TypeOf(localType) != reflect.TypeOf(targetType) { + return false, nil + } + + switch lv := (localType).(type) { + case *Void, *Struct, *Union, *Enum, *Fwd: + // Nothing to do here + + case *Int: + tv := targetType.(*Int) + if lv.isBitfield() || tv.isBitfield() { + return false, nil + } + + case *Pointer, *Array: + depth++ + localType.walk(&localTs) + targetType.walk(&targetTs) + + case *FuncProto: + tv := targetType.(*FuncProto) + if len(lv.Params) != len(tv.Params) { + return false, nil + } + + depth++ + localType.walk(&localTs) + targetType.walk(&targetTs) + + default: + return false, fmt.Errorf("unsupported type %T", localType) + } + } + + if l != nil { + return false, fmt.Errorf("dangling local type %T", *l) + } + + if t != nil { + return false, fmt.Errorf("dangling target type %T", *t) + } + + return true, nil +} + +/* The comment below is from bpf_core_fields_are_compat in libbpf.c: + * + * Check two types for compatibility for the purpose of field access + * relocation. const/volatile/restrict and typedefs are skipped to ensure we + * are relocating semantically compatible entities: + * - any two STRUCTs/UNIONs are compatible and can be mixed; + * - any two FWDs are compatible, if their names match (modulo flavor suffix); + * - any two PTRs are always compatible; + * - for ENUMs, names should be the same (ignoring flavor suffix) or at + * least one of enums should be anonymous; + * - for ENUMs, check sizes, names are ignored; + * - for INT, size and signedness are ignored; + * - for ARRAY, dimensionality is ignored, element types are checked for + * compatibility recursively; + * - everything else shouldn't be ever a target of relocation. + * These rules are not set in stone and probably will be adjusted as we get + * more experience with using BPF CO-RE relocations. + */ +func coreAreMembersCompatible(localType Type, targetType Type) (bool, error) { + doNamesMatch := func(a, b string) bool { + if a == "" || b == "" { + // allow anonymous and named type to match + return true + } + + return essentialName(a) == essentialName(b) + } + + for depth := 0; depth <= maxTypeDepth; depth++ { + localType = skipQualifierAndTypedef(localType) + targetType = skipQualifierAndTypedef(targetType) + + _, lok := localType.(composite) + _, tok := targetType.(composite) + if lok && tok { + return true, nil + } + + if reflect.TypeOf(localType) != reflect.TypeOf(targetType) { + return false, nil + } + + switch lv := localType.(type) { + case *Pointer: + return true, nil + + case *Enum: + tv := targetType.(*Enum) + return doNamesMatch(lv.name(), tv.name()), nil + + case *Fwd: + tv := targetType.(*Fwd) + return doNamesMatch(lv.name(), tv.name()), nil + + case *Int: + tv := targetType.(*Int) + return !lv.isBitfield() && !tv.isBitfield(), nil + + case *Array: + tv := targetType.(*Array) + + localType = lv.Type + targetType = tv.Type + + default: + return false, fmt.Errorf("unsupported type %T", localType) + } + } + + return false, errors.New("types are nested too deep") +} + +func skipQualifierAndTypedef(typ Type) Type { + result := typ + for depth := 0; depth <= maxTypeDepth; depth++ { + switch v := (result).(type) { + case qualifier: + result = v.qualify() + case *Typedef: + result = v.Type + default: + return result + } + } + return typ +} diff --git a/vendor/github.com/cilium/ebpf/internal/btf/ext_info.go b/vendor/github.com/cilium/ebpf/internal/btf/ext_info.go index 28918ae9e0c8e..6a21b6bda5ccf 100644 --- a/vendor/github.com/cilium/ebpf/internal/btf/ext_info.go +++ b/vendor/github.com/cilium/ebpf/internal/btf/ext_info.go @@ -1,6 +1,7 @@ package btf import ( + "bufio" "bytes" "encoding/binary" "errors" @@ -24,55 +25,82 @@ type btfExtHeader struct { LineInfoLen uint32 } -func parseExtInfos(r io.ReadSeeker, bo binary.ByteOrder, strings stringTable) (funcInfo, lineInfo map[string]extInfo, err error) { +type btfExtCoreHeader struct { + CoreReloOff uint32 + CoreReloLen uint32 +} + +func parseExtInfos(r io.ReadSeeker, bo binary.ByteOrder, strings stringTable) (funcInfo, lineInfo map[string]extInfo, coreRelos map[string]bpfCoreRelos, err error) { var header btfExtHeader + var coreHeader btfExtCoreHeader if err := binary.Read(r, bo, &header); err != nil { - return nil, nil, fmt.Errorf("can't read header: %v", err) + return nil, nil, nil, fmt.Errorf("can't read header: %v", err) } if header.Magic != btfMagic { - return nil, nil, fmt.Errorf("incorrect magic value %v", header.Magic) + return nil, nil, nil, fmt.Errorf("incorrect magic value %v", header.Magic) } if header.Version != 1 { - return nil, nil, fmt.Errorf("unexpected version %v", header.Version) + return nil, nil, nil, fmt.Errorf("unexpected version %v", header.Version) } if header.Flags != 0 { - return nil, nil, fmt.Errorf("unsupported flags %v", header.Flags) + return nil, nil, nil, fmt.Errorf("unsupported flags %v", header.Flags) } remainder := int64(header.HdrLen) - int64(binary.Size(&header)) if remainder < 0 { - return nil, nil, errors.New("header is too short") + return nil, nil, nil, errors.New("header is too short") + } + + coreHdrSize := int64(binary.Size(&coreHeader)) + if remainder >= coreHdrSize { + if err := binary.Read(r, bo, &coreHeader); err != nil { + return nil, nil, nil, fmt.Errorf("can't read CO-RE relocation header: %v", err) + } + remainder -= coreHdrSize } // Of course, the .BTF.ext header has different semantics than the // .BTF ext header. We need to ignore non-null values. _, err = io.CopyN(ioutil.Discard, r, remainder) if err != nil { - return nil, nil, fmt.Errorf("header padding: %v", err) + return nil, nil, nil, fmt.Errorf("header padding: %v", err) } if _, err := r.Seek(int64(header.HdrLen+header.FuncInfoOff), io.SeekStart); err != nil { - return nil, nil, fmt.Errorf("can't seek to function info section: %v", err) + return nil, nil, nil, fmt.Errorf("can't seek to function info section: %v", err) } - funcInfo, err = parseExtInfo(io.LimitReader(r, int64(header.FuncInfoLen)), bo, strings) + buf := bufio.NewReader(io.LimitReader(r, int64(header.FuncInfoLen))) + funcInfo, err = parseExtInfo(buf, bo, strings) if err != nil { - return nil, nil, fmt.Errorf("function info: %w", err) + return nil, nil, nil, fmt.Errorf("function info: %w", err) } if _, err := r.Seek(int64(header.HdrLen+header.LineInfoOff), io.SeekStart); err != nil { - return nil, nil, fmt.Errorf("can't seek to line info section: %v", err) + return nil, nil, nil, fmt.Errorf("can't seek to line info section: %v", err) } - lineInfo, err = parseExtInfo(io.LimitReader(r, int64(header.LineInfoLen)), bo, strings) + buf = bufio.NewReader(io.LimitReader(r, int64(header.LineInfoLen))) + lineInfo, err = parseExtInfo(buf, bo, strings) if err != nil { - return nil, nil, fmt.Errorf("line info: %w", err) + return nil, nil, nil, fmt.Errorf("line info: %w", err) + } + + if coreHeader.CoreReloOff > 0 && coreHeader.CoreReloLen > 0 { + if _, err := r.Seek(int64(header.HdrLen+coreHeader.CoreReloOff), io.SeekStart); err != nil { + return nil, nil, nil, fmt.Errorf("can't seek to CO-RE relocation section: %v", err) + } + + coreRelos, err = parseExtInfoRelos(io.LimitReader(r, int64(coreHeader.CoreReloLen)), bo, strings) + if err != nil { + return nil, nil, nil, fmt.Errorf("CO-RE relocation info: %w", err) + } } - return funcInfo, lineInfo, nil + return funcInfo, lineInfo, coreRelos, nil } type btfExtInfoSec struct { @@ -127,6 +155,8 @@ func (ei extInfo) MarshalBinary() ([]byte, error) { } func parseExtInfo(r io.Reader, bo binary.ByteOrder, strings stringTable) (map[string]extInfo, error) { + const maxRecordSize = 256 + var recordSize uint32 if err := binary.Read(r, bo, &recordSize); err != nil { return nil, fmt.Errorf("can't read record size: %v", err) @@ -136,23 +166,15 @@ func parseExtInfo(r io.Reader, bo binary.ByteOrder, strings stringTable) (map[st // Need at least insnOff return nil, errors.New("record size too short") } + if recordSize > maxRecordSize { + return nil, fmt.Errorf("record size %v exceeds %v", recordSize, maxRecordSize) + } result := make(map[string]extInfo) for { - var infoHeader btfExtInfoSec - if err := binary.Read(r, bo, &infoHeader); err == io.EOF { + secName, infoHeader, err := parseExtInfoHeader(r, bo, strings) + if errors.Is(err, io.EOF) { return result, nil - } else if err != nil { - return nil, fmt.Errorf("can't read ext info header: %v", err) - } - - secName, err := strings.Lookup(infoHeader.SecNameOff) - if err != nil { - return nil, fmt.Errorf("can't get section name: %w", err) - } - - if infoHeader.NumInfo == 0 { - return nil, fmt.Errorf("section %s has invalid number of records", secName) } var records []extInfoRecord @@ -180,3 +202,80 @@ func parseExtInfo(r io.Reader, bo binary.ByteOrder, strings stringTable) (map[st } } } + +// bpfCoreRelo matches `struct bpf_core_relo` from the kernel +type bpfCoreRelo struct { + InsnOff uint32 + TypeID TypeID + AccessStrOff uint32 + ReloKind coreReloKind +} + +type bpfCoreRelos []bpfCoreRelo + +// append two slices of extInfoRelo to each other. The InsnOff of b are adjusted +// by offset. +func (r bpfCoreRelos) append(other bpfCoreRelos, offset uint64) bpfCoreRelos { + result := make([]bpfCoreRelo, 0, len(r)+len(other)) + result = append(result, r...) + for _, relo := range other { + relo.InsnOff += uint32(offset) + result = append(result, relo) + } + return result +} + +var extInfoReloSize = binary.Size(bpfCoreRelo{}) + +func parseExtInfoRelos(r io.Reader, bo binary.ByteOrder, strings stringTable) (map[string]bpfCoreRelos, error) { + var recordSize uint32 + if err := binary.Read(r, bo, &recordSize); err != nil { + return nil, fmt.Errorf("read record size: %v", err) + } + + if recordSize != uint32(extInfoReloSize) { + return nil, fmt.Errorf("expected record size %d, got %d", extInfoReloSize, recordSize) + } + + result := make(map[string]bpfCoreRelos) + for { + secName, infoHeader, err := parseExtInfoHeader(r, bo, strings) + if errors.Is(err, io.EOF) { + return result, nil + } + + var relos []bpfCoreRelo + for i := uint32(0); i < infoHeader.NumInfo; i++ { + var relo bpfCoreRelo + if err := binary.Read(r, bo, &relo); err != nil { + return nil, fmt.Errorf("section %v: read record: %v", secName, err) + } + + if relo.InsnOff%asm.InstructionSize != 0 { + return nil, fmt.Errorf("section %v: offset %v is not aligned with instruction size", secName, relo.InsnOff) + } + + relos = append(relos, relo) + } + + result[secName] = relos + } +} + +func parseExtInfoHeader(r io.Reader, bo binary.ByteOrder, strings stringTable) (string, *btfExtInfoSec, error) { + var infoHeader btfExtInfoSec + if err := binary.Read(r, bo, &infoHeader); err != nil { + return "", nil, fmt.Errorf("read ext info header: %w", err) + } + + secName, err := strings.Lookup(infoHeader.SecNameOff) + if err != nil { + return "", nil, fmt.Errorf("get section name: %w", err) + } + + if infoHeader.NumInfo == 0 { + return "", nil, fmt.Errorf("section %s has zero records", secName) + } + + return secName, &infoHeader, nil +} diff --git a/vendor/github.com/cilium/ebpf/internal/btf/fuzz.go b/vendor/github.com/cilium/ebpf/internal/btf/fuzz.go new file mode 100644 index 0000000000000..37e043fd378c7 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/internal/btf/fuzz.go @@ -0,0 +1,49 @@ +// +build gofuzz + +// Use with https://github.com/dvyukov/go-fuzz + +package btf + +import ( + "bytes" + "encoding/binary" + + "github.com/cilium/ebpf/internal" +) + +func FuzzSpec(data []byte) int { + if len(data) < binary.Size(btfHeader{}) { + return -1 + } + + spec, err := loadNakedSpec(bytes.NewReader(data), internal.NativeEndian, nil, nil) + if err != nil { + if spec != nil { + panic("spec is not nil") + } + return 0 + } + if spec == nil { + panic("spec is nil") + } + return 1 +} + +func FuzzExtInfo(data []byte) int { + if len(data) < binary.Size(btfExtHeader{}) { + return -1 + } + + table := stringTable("\x00foo\x00barfoo\x00") + info, err := parseExtInfo(bytes.NewReader(data), internal.NativeEndian, table) + if err != nil { + if info != nil { + panic("info is not nil") + } + return 0 + } + if info == nil { + panic("info is nil") + } + return 1 +} diff --git a/vendor/github.com/cilium/ebpf/internal/btf/types.go b/vendor/github.com/cilium/ebpf/internal/btf/types.go index 264142abc0ad6..9e1fd8d0b2dff 100644 --- a/vendor/github.com/cilium/ebpf/internal/btf/types.go +++ b/vendor/github.com/cilium/ebpf/internal/btf/types.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "math" + "strings" ) const maxTypeDepth = 32 @@ -20,10 +21,22 @@ func (tid TypeID) ID() TypeID { type Type interface { ID() TypeID + String() string + // Make a copy of the type, without copying Type members. copy() Type - walk(*copyStack) + // Enumerate all nested Types. Repeated calls must visit nested + // types in the same order. + walk(*typeDeque) +} + +// namedType is a type with a name. +// +// Most named types simply embed Name. +type namedType interface { + Type + name() string } // Name identifies a type. @@ -39,9 +52,18 @@ func (n Name) name() string { type Void struct{} func (v *Void) ID() TypeID { return 0 } +func (v *Void) String() string { return "void#0" } func (v *Void) size() uint32 { return 0 } func (v *Void) copy() Type { return (*Void)(nil) } -func (v *Void) walk(*copyStack) {} +func (v *Void) walk(*typeDeque) {} + +type IntEncoding byte + +const ( + Signed IntEncoding = 1 << iota + Char + Bool +) // Int is an integer of a given length. type Int struct { @@ -49,24 +71,64 @@ type Int struct { Name // The size of the integer in bytes. - Size uint32 + Size uint32 + Encoding IntEncoding + // Offset is the starting bit offset. Currently always 0. + // See https://www.kernel.org/doc/html/latest/bpf/btf.html#btf-kind-int + Offset uint32 + Bits byte +} + +var _ namedType = (*Int)(nil) + +func (i *Int) String() string { + var s strings.Builder + + switch { + case i.Encoding&Char != 0: + s.WriteString("char") + case i.Encoding&Bool != 0: + s.WriteString("bool") + default: + if i.Encoding&Signed == 0 { + s.WriteRune('u') + } + s.WriteString("int") + fmt.Fprintf(&s, "%d", i.Size*8) + } + + fmt.Fprintf(&s, "#%d", i.TypeID) + + if i.Bits > 0 { + fmt.Fprintf(&s, "[bits=%d]", i.Bits) + } + + return s.String() } func (i *Int) size() uint32 { return i.Size } -func (i *Int) walk(*copyStack) {} +func (i *Int) walk(*typeDeque) {} func (i *Int) copy() Type { cpy := *i return &cpy } +func (i *Int) isBitfield() bool { + return i.Offset > 0 +} + // Pointer is a pointer to another type. type Pointer struct { TypeID Target Type } -func (p *Pointer) size() uint32 { return 8 } -func (p *Pointer) walk(cs *copyStack) { cs.push(&p.Target) } +func (p *Pointer) String() string { + return fmt.Sprintf("pointer#%d[target=#%d]", p.TypeID, p.Target.ID()) +} + +func (p *Pointer) size() uint32 { return 8 } +func (p *Pointer) walk(tdq *typeDeque) { tdq.push(&p.Target) } func (p *Pointer) copy() Type { cpy := *p return &cpy @@ -79,7 +141,11 @@ type Array struct { Nelems uint32 } -func (arr *Array) walk(cs *copyStack) { cs.push(&arr.Type) } +func (arr *Array) String() string { + return fmt.Sprintf("array#%d[type=#%d n=%d]", arr.TypeID, arr.Type.ID(), arr.Nelems) +} + +func (arr *Array) walk(tdq *typeDeque) { tdq.push(&arr.Type) } func (arr *Array) copy() Type { cpy := *arr return &cpy @@ -94,11 +160,15 @@ type Struct struct { Members []Member } +func (s *Struct) String() string { + return fmt.Sprintf("struct#%d[%q]", s.TypeID, s.Name) +} + func (s *Struct) size() uint32 { return s.Size } -func (s *Struct) walk(cs *copyStack) { +func (s *Struct) walk(tdq *typeDeque) { for i := range s.Members { - cs.push(&s.Members[i].Type) + tdq.push(&s.Members[i].Type) } } @@ -109,6 +179,10 @@ func (s *Struct) copy() Type { return &cpy } +func (s *Struct) members() []Member { + return s.Members +} + // Union is a compound type where members occupy the same memory. type Union struct { TypeID @@ -118,11 +192,15 @@ type Union struct { Members []Member } +func (u *Union) String() string { + return fmt.Sprintf("union#%d[%q]", u.TypeID, u.Name) +} + func (u *Union) size() uint32 { return u.Size } -func (u *Union) walk(cs *copyStack) { +func (u *Union) walk(tdq *typeDeque) { for i := range u.Members { - cs.push(&u.Members[i].Type) + tdq.push(&u.Members[i].Type) } } @@ -133,35 +211,90 @@ func (u *Union) copy() Type { return &cpy } +func (u *Union) members() []Member { + return u.Members +} + +type composite interface { + members() []Member +} + +var ( + _ composite = (*Struct)(nil) + _ composite = (*Union)(nil) +) + // Member is part of a Struct or Union. // // It is not a valid Type. type Member struct { Name - Type Type - Offset uint32 + Type Type + // Offset is the bit offset of this member + Offset uint32 + BitfieldSize uint32 } // Enum lists possible values. type Enum struct { TypeID Name + Values []EnumValue +} + +func (e *Enum) String() string { + return fmt.Sprintf("enum#%d[%q]", e.TypeID, e.Name) +} + +// EnumValue is part of an Enum +// +// Is is not a valid Type +type EnumValue struct { + Name + Value int32 } func (e *Enum) size() uint32 { return 4 } -func (e *Enum) walk(*copyStack) {} +func (e *Enum) walk(*typeDeque) {} func (e *Enum) copy() Type { cpy := *e + cpy.Values = make([]EnumValue, len(e.Values)) + copy(cpy.Values, e.Values) return &cpy } +// FwdKind is the type of forward declaration. +type FwdKind int + +// Valid types of forward declaration. +const ( + FwdStruct FwdKind = iota + FwdUnion +) + +func (fk FwdKind) String() string { + switch fk { + case FwdStruct: + return "struct" + case FwdUnion: + return "union" + default: + return fmt.Sprintf("%T(%d)", fk, int(fk)) + } +} + // Fwd is a forward declaration of a Type. type Fwd struct { TypeID Name + Kind FwdKind +} + +func (f *Fwd) String() string { + return fmt.Sprintf("fwd#%d[%s %q]", f.TypeID, f.Kind, f.Name) } -func (f *Fwd) walk(*copyStack) {} +func (f *Fwd) walk(*typeDeque) {} func (f *Fwd) copy() Type { cpy := *f return &cpy @@ -174,43 +307,62 @@ type Typedef struct { Type Type } -func (td *Typedef) walk(cs *copyStack) { cs.push(&td.Type) } +func (td *Typedef) String() string { + return fmt.Sprintf("typedef#%d[%q #%d]", td.TypeID, td.Name, td.Type.ID()) +} + +func (td *Typedef) walk(tdq *typeDeque) { tdq.push(&td.Type) } func (td *Typedef) copy() Type { cpy := *td return &cpy } -// Volatile is a modifier. +// Volatile is a qualifier. type Volatile struct { TypeID Type Type } -func (v *Volatile) walk(cs *copyStack) { cs.push(&v.Type) } +func (v *Volatile) String() string { + return fmt.Sprintf("volatile#%d[#%d]", v.TypeID, v.Type.ID()) +} + +func (v *Volatile) qualify() Type { return v.Type } +func (v *Volatile) walk(tdq *typeDeque) { tdq.push(&v.Type) } func (v *Volatile) copy() Type { cpy := *v return &cpy } -// Const is a modifier. +// Const is a qualifier. type Const struct { TypeID Type Type } -func (c *Const) walk(cs *copyStack) { cs.push(&c.Type) } +func (c *Const) String() string { + return fmt.Sprintf("const#%d[#%d]", c.TypeID, c.Type.ID()) +} + +func (c *Const) qualify() Type { return c.Type } +func (c *Const) walk(tdq *typeDeque) { tdq.push(&c.Type) } func (c *Const) copy() Type { cpy := *c return &cpy } -// Restrict is a modifier. +// Restrict is a qualifier. type Restrict struct { TypeID Type Type } -func (r *Restrict) walk(cs *copyStack) { cs.push(&r.Type) } +func (r *Restrict) String() string { + return fmt.Sprintf("restrict#%d[#%d]", r.TypeID, r.Type.ID()) +} + +func (r *Restrict) qualify() Type { return r.Type } +func (r *Restrict) walk(tdq *typeDeque) { tdq.push(&r.Type) } func (r *Restrict) copy() Type { cpy := *r return &cpy @@ -223,7 +375,11 @@ type Func struct { Type Type } -func (f *Func) walk(cs *copyStack) { cs.push(&f.Type) } +func (f *Func) String() string { + return fmt.Sprintf("func#%d[%q proto=#%d]", f.TypeID, f.Name, f.Type.ID()) +} + +func (f *Func) walk(tdq *typeDeque) { tdq.push(&f.Type) } func (f *Func) copy() Type { cpy := *f return &cpy @@ -233,15 +389,38 @@ func (f *Func) copy() Type { type FuncProto struct { TypeID Return Type - // Parameters not supported yet + Params []FuncParam +} + +func (fp *FuncProto) String() string { + var s strings.Builder + fmt.Fprintf(&s, "proto#%d[", fp.TypeID) + for _, param := range fp.Params { + fmt.Fprintf(&s, "%q=#%d, ", param.Name, param.Type.ID()) + } + fmt.Fprintf(&s, "return=#%d]", fp.Return.ID()) + return s.String() +} + +func (fp *FuncProto) walk(tdq *typeDeque) { + tdq.push(&fp.Return) + for i := range fp.Params { + tdq.push(&fp.Params[i].Type) + } } -func (fp *FuncProto) walk(cs *copyStack) { cs.push(&fp.Return) } func (fp *FuncProto) copy() Type { cpy := *fp + cpy.Params = make([]FuncParam, len(fp.Params)) + copy(cpy.Params, fp.Params) return &cpy } +type FuncParam struct { + Name + Type Type +} + // Var is a global variable. type Var struct { TypeID @@ -249,7 +428,12 @@ type Var struct { Type Type } -func (v *Var) walk(cs *copyStack) { cs.push(&v.Type) } +func (v *Var) String() string { + // TODO: Linkage + return fmt.Sprintf("var#%d[%q]", v.TypeID, v.Name) +} + +func (v *Var) walk(tdq *typeDeque) { tdq.push(&v.Type) } func (v *Var) copy() Type { cpy := *v return &cpy @@ -263,11 +447,15 @@ type Datasec struct { Vars []VarSecinfo } +func (ds *Datasec) String() string { + return fmt.Sprintf("section#%d[%q]", ds.TypeID, ds.Name) +} + func (ds *Datasec) size() uint32 { return ds.Size } -func (ds *Datasec) walk(cs *copyStack) { +func (ds *Datasec) walk(tdq *typeDeque) { for i := range ds.Vars { - cs.push(&ds.Vars[i].Type) + tdq.push(&ds.Vars[i].Type) } } @@ -279,6 +467,8 @@ func (ds *Datasec) copy() Type { } // VarSecinfo describes variable in a Datasec +// +// It is not a valid Type. type VarSecinfo struct { Type Type Offset uint32 @@ -298,6 +488,16 @@ var ( _ sizer = (*Datasec)(nil) ) +type qualifier interface { + qualify() Type +} + +var ( + _ qualifier = (*Const)(nil) + _ qualifier = (*Restrict)(nil) + _ qualifier = (*Volatile)(nil) +) + // Sizeof returns the size of a type in bytes. // // Returns an error if the size can't be computed. @@ -326,14 +526,9 @@ func Sizeof(typ Type) (int, error) { case *Typedef: typ = v.Type continue - case *Volatile: - typ = v.Type - continue - case *Const: - typ = v.Type - continue - case *Restrict: - typ = v.Type + + case qualifier: + typ = v.qualify() continue default: @@ -361,7 +556,7 @@ func Sizeof(typ Type) (int, error) { func copyType(typ Type) Type { var ( copies = make(map[Type]Type) - work copyStack + work typeDeque ) for t := &typ; t != nil; t = work.pop() { @@ -382,40 +577,83 @@ func copyType(typ Type) Type { return typ } -// copyStack keeps track of pointers to types which still -// need to be copied. -type copyStack []*Type +// typeDeque keeps track of pointers to types which still +// need to be visited. +type typeDeque struct { + types []*Type + read, write uint64 + mask uint64 +} // push adds a type to the stack. -func (cs *copyStack) push(t *Type) { - *cs = append(*cs, t) +func (dq *typeDeque) push(t *Type) { + if dq.write-dq.read < uint64(len(dq.types)) { + dq.types[dq.write&dq.mask] = t + dq.write++ + return + } + + new := len(dq.types) * 2 + if new == 0 { + new = 8 + } + + types := make([]*Type, new) + pivot := dq.read & dq.mask + n := copy(types, dq.types[pivot:]) + n += copy(types[n:], dq.types[:pivot]) + types[n] = t + + dq.types = types + dq.mask = uint64(new) - 1 + dq.read, dq.write = 0, uint64(n+1) } -// pop returns the topmost Type, or nil. -func (cs *copyStack) pop() *Type { - n := len(*cs) - if n == 0 { +// shift returns the first element or null. +func (dq *typeDeque) shift() *Type { + if dq.read == dq.write { return nil } - t := (*cs)[n-1] - *cs = (*cs)[:n-1] + index := dq.read & dq.mask + t := dq.types[index] + dq.types[index] = nil + dq.read++ return t } -type namer interface { - name() string +// pop returns the last element or null. +func (dq *typeDeque) pop() *Type { + if dq.read == dq.write { + return nil + } + + dq.write-- + index := dq.write & dq.mask + t := dq.types[index] + dq.types[index] = nil + return t } -var _ namer = Name("") +// all returns all elements. +// +// The deque is empty after calling this method. +func (dq *typeDeque) all() []*Type { + length := dq.write - dq.read + types := make([]*Type, 0, length) + for t := dq.shift(); t != nil; t = dq.shift() { + types = append(types, t) + } + return types +} // inflateRawTypes takes a list of raw btf types linked via type IDs, and turns // it into a graph of Types connected via pointers. // -// Returns a map of named types (so, where NameOff is non-zero). Since BTF ignores -// compilation units, multiple types may share the same name. A Type may form a -// cyclic graph by pointing at itself. -func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (namedTypes map[string][]Type, err error) { +// Returns a map of named types (so, where NameOff is non-zero) and a slice of types +// indexed by TypeID. Since BTF ignores compilation units, multiple types may share +// the same name. A Type may form a cyclic graph by pointing at itself. +func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (types []Type, namedTypes map[string][]namedType, err error) { type fixupDef struct { id TypeID expectedKind btfKind @@ -427,7 +665,7 @@ func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (namedTypes map fixups = append(fixups, fixupDef{id, expectedKind, typ}) } - convertMembers := func(raw []btfMember) ([]Member, error) { + convertMembers := func(raw []btfMember, kindFlag bool) ([]Member, error) { // NB: The fixup below relies on pre-allocating this array to // work, since otherwise append might re-allocate members. members := make([]Member, 0, len(raw)) @@ -436,10 +674,15 @@ func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (namedTypes map if err != nil { return nil, fmt.Errorf("can't get name for member %d: %w", i, err) } - members = append(members, Member{ + m := Member{ Name: name, Offset: btfMember.Offset, - }) + } + if kindFlag { + m.BitfieldSize = btfMember.Offset >> 24 + m.Offset &= 0xffffff + } + members = append(members, m) } for i := range members { fixup(raw[i].Type, kindUnknown, &members[i].Type) @@ -447,9 +690,9 @@ func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (namedTypes map return members, nil } - types := make([]Type, 0, len(rawTypes)) + types = make([]Type, 0, len(rawTypes)) types = append(types, (*Void)(nil)) - namedTypes = make(map[string][]Type) + namedTypes = make(map[string][]namedType) for i, raw := range rawTypes { var ( @@ -461,12 +704,13 @@ func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (namedTypes map name, err := rawStrings.LookupName(raw.NameOff) if err != nil { - return nil, fmt.Errorf("can't get name for type id %d: %w", id, err) + return nil, nil, fmt.Errorf("get name for type id %d: %w", id, err) } switch raw.Kind() { case kindInt: - typ = &Int{id, name, raw.Size()} + encoding, offset, bits := intEncoding(*raw.data.(*uint32)) + typ = &Int{id, name, raw.Size(), encoding, offset, bits} case kindPointer: ptr := &Pointer{id, nil} @@ -483,24 +727,40 @@ func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (namedTypes map typ = arr case kindStruct: - members, err := convertMembers(raw.data.([]btfMember)) + members, err := convertMembers(raw.data.([]btfMember), raw.KindFlag()) if err != nil { - return nil, fmt.Errorf("struct %s (id %d): %w", name, id, err) + return nil, nil, fmt.Errorf("struct %s (id %d): %w", name, id, err) } typ = &Struct{id, name, raw.Size(), members} case kindUnion: - members, err := convertMembers(raw.data.([]btfMember)) + members, err := convertMembers(raw.data.([]btfMember), raw.KindFlag()) if err != nil { - return nil, fmt.Errorf("union %s (id %d): %w", name, id, err) + return nil, nil, fmt.Errorf("union %s (id %d): %w", name, id, err) } typ = &Union{id, name, raw.Size(), members} case kindEnum: - typ = &Enum{id, name} + rawvals := raw.data.([]btfEnum) + vals := make([]EnumValue, 0, len(rawvals)) + for i, btfVal := range rawvals { + name, err := rawStrings.LookupName(btfVal.NameOff) + if err != nil { + return nil, nil, fmt.Errorf("get name for enum value %d: %s", i, err) + } + vals = append(vals, EnumValue{ + Name: name, + Value: btfVal.Val, + }) + } + typ = &Enum{id, name, vals} case kindForward: - typ = &Fwd{id, name} + if raw.KindFlag() { + typ = &Fwd{id, name, FwdUnion} + } else { + typ = &Fwd{id, name, FwdStruct} + } case kindTypedef: typedef := &Typedef{id, name, nil} @@ -528,7 +788,22 @@ func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (namedTypes map typ = fn case kindFuncProto: - fp := &FuncProto{id, nil} + rawparams := raw.data.([]btfParam) + params := make([]FuncParam, 0, len(rawparams)) + for i, param := range rawparams { + name, err := rawStrings.LookupName(param.NameOff) + if err != nil { + return nil, nil, fmt.Errorf("get name for func proto parameter %d: %s", i, err) + } + params = append(params, FuncParam{ + Name: name, + }) + } + for i := range params { + fixup(rawparams[i].Type, kindUnknown, ¶ms[i].Type) + } + + fp := &FuncProto{id, nil, params} fixup(raw.Type(), kindUnknown, &fp.Return) typ = fp @@ -552,14 +827,14 @@ func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (namedTypes map typ = &Datasec{id, name, raw.SizeType, vars} default: - return nil, fmt.Errorf("type id %d: unknown kind: %v", id, raw.Kind()) + return nil, nil, fmt.Errorf("type id %d: unknown kind: %v", id, raw.Kind()) } types = append(types, typ) - if namer, ok := typ.(namer); ok { - if name := namer.name(); name != "" { - namedTypes[name] = append(namedTypes[name], typ) + if named, ok := typ.(namedType); ok { + if name := essentialName(named.name()); name != "" { + namedTypes[name] = append(namedTypes[name], named) } } } @@ -567,7 +842,7 @@ func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (namedTypes map for _, fixup := range fixups { i := int(fixup.id) if i >= len(types) { - return nil, fmt.Errorf("reference to invalid type id: %d", fixup.id) + return nil, nil, fmt.Errorf("reference to invalid type id: %d", fixup.id) } // Default void (id 0) to unknown @@ -577,11 +852,20 @@ func inflateRawTypes(rawTypes []rawType, rawStrings stringTable) (namedTypes map } if expected := fixup.expectedKind; expected != kindUnknown && rawKind != expected { - return nil, fmt.Errorf("expected type id %d to have kind %s, found %s", fixup.id, expected, rawKind) + return nil, nil, fmt.Errorf("expected type id %d to have kind %s, found %s", fixup.id, expected, rawKind) } *fixup.typ = types[i] } - return namedTypes, nil + return types, namedTypes, nil +} + +// essentialName returns name without a ___ suffix. +func essentialName(name string) string { + lastIdx := strings.LastIndex(name, "___") + if lastIdx > 0 { + return name[:lastIdx] + } + return name } diff --git a/vendor/github.com/cilium/ebpf/internal/elf.go b/vendor/github.com/cilium/ebpf/internal/elf.go new file mode 100644 index 0000000000000..c3f9ea0f8a405 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/internal/elf.go @@ -0,0 +1,52 @@ +package internal + +import ( + "debug/elf" + "fmt" + "io" +) + +type SafeELFFile struct { + *elf.File +} + +// NewSafeELFFile reads an ELF safely. +// +// Any panic during parsing is turned into an error. This is necessary since +// there are a bunch of unfixed bugs in debug/elf. +// +// https://github.com/golang/go/issues?q=is%3Aissue+is%3Aopen+debug%2Felf+in%3Atitle +func NewSafeELFFile(r io.ReaderAt) (safe *SafeELFFile, err error) { + defer func() { + r := recover() + if r == nil { + return + } + + safe = nil + err = fmt.Errorf("reading ELF file panicked: %s", r) + }() + + file, err := elf.NewFile(r) + if err != nil { + return nil, err + } + + return &SafeELFFile{file}, nil +} + +// Symbols is the safe version of elf.File.Symbols. +func (se *SafeELFFile) Symbols() (syms []elf.Symbol, err error) { + defer func() { + r := recover() + if r == nil { + return + } + + syms = nil + err = fmt.Errorf("reading ELF symbols panicked: %s", r) + }() + + syms, err = se.File.Symbols() + return +} diff --git a/vendor/github.com/cilium/ebpf/internal/feature.go b/vendor/github.com/cilium/ebpf/internal/feature.go index 7375b21ef984a..ec62ed39b7da6 100644 --- a/vendor/github.com/cilium/ebpf/internal/feature.go +++ b/vendor/github.com/cilium/ebpf/internal/feature.go @@ -20,6 +20,9 @@ type UnsupportedFeatureError struct { } func (ufe *UnsupportedFeatureError) Error() string { + if ufe.MinimumVersion.Unspecified() { + return fmt.Sprintf("%s not supported", ufe.Name) + } return fmt.Sprintf("%s not supported (requires >= %s)", ufe.Name, ufe.MinimumVersion) } @@ -29,7 +32,7 @@ func (ufe *UnsupportedFeatureError) Is(target error) bool { } type featureTest struct { - sync.Mutex + sync.RWMutex successful bool result error } @@ -39,10 +42,10 @@ type featureTest struct { // // The return values have the following semantics: // +// err == ErrNotSupported: the feature is not available +// err == nil: the feature is available // err != nil: the test couldn't be executed -// err == nil && available: the feature is available -// err == nil && !available: the feature isn't available -type FeatureTestFn func() (available bool, err error) +type FeatureTestFn func() error // FeatureTest wraps a function so that it is run at most once. // @@ -58,32 +61,40 @@ func FeatureTest(name, version string, fn FeatureTestFn) func() error { ft := new(featureTest) return func() error { + ft.RLock() + if ft.successful { + defer ft.RUnlock() + return ft.result + } + ft.RUnlock() ft.Lock() defer ft.Unlock() - + // check one more time on the off + // chance that two go routines + // were able to call into the write + // lock if ft.successful { return ft.result } + err := fn() + switch { + case errors.Is(err, ErrNotSupported): + ft.result = &UnsupportedFeatureError{ + MinimumVersion: v, + Name: name, + } + fallthrough - available, err := fn() - if errors.Is(err, ErrNotSupported) { - // The feature test aborted because a dependent feature - // is missing, which we should cache. - available = false - } else if err != nil { + case err == nil: + ft.successful = true + + default: // We couldn't execute the feature test to a point // where it could make a determination. // Don't cache the result, just return it. - return fmt.Errorf("can't detect support for %s: %w", name, err) + return fmt.Errorf("detect support for %s: %w", name, err) } - ft.successful = true - if !available { - ft.result = &UnsupportedFeatureError{ - MinimumVersion: v, - Name: name, - } - } return ft.result } } @@ -120,3 +131,8 @@ func (v Version) Less(other Version) bool { } return false } + +// Unspecified returns true if the version is all zero. +func (v Version) Unspecified() bool { + return v[0] == 0 && v[1] == 0 && v[2] == 0 +} diff --git a/vendor/github.com/cilium/ebpf/internal/syscall.go b/vendor/github.com/cilium/ebpf/internal/syscall.go index efbf40327c6b8..c808151312ae7 100644 --- a/vendor/github.com/cilium/ebpf/internal/syscall.go +++ b/vendor/github.com/cilium/ebpf/internal/syscall.go @@ -91,6 +91,19 @@ func BPFProgDetach(attr *BPFProgDetachAttr) error { return err } +type BPFEnableStatsAttr struct { + StatsType uint32 +} + +func BPFEnableStats(attr *BPFEnableStatsAttr) (*FD, error) { + ptr, err := BPF(BPF_ENABLE_STATS, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + if err != nil { + return nil, fmt.Errorf("enable stats: %w", err) + } + return NewFD(uint32(ptr)), nil + +} + type bpfObjAttr struct { fileName Pointer fd uint32 @@ -137,3 +150,30 @@ func BPFObjGet(fileName string) (*FD, error) { } return NewFD(uint32(ptr)), nil } + +type bpfObjGetInfoByFDAttr struct { + fd uint32 + infoLen uint32 + info Pointer +} + +// BPFObjGetInfoByFD wraps BPF_OBJ_GET_INFO_BY_FD. +// +// Available from 4.13. +func BPFObjGetInfoByFD(fd *FD, info unsafe.Pointer, size uintptr) error { + value, err := fd.Value() + if err != nil { + return err + } + + attr := bpfObjGetInfoByFDAttr{ + fd: value, + infoLen: uint32(size), + info: NewPointer(info), + } + _, err = BPF(BPF_OBJ_GET_INFO_BY_FD, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) + if err != nil { + return fmt.Errorf("fd %v: %w", fd, err) + } + return nil +} diff --git a/vendor/github.com/cilium/ebpf/internal/unix/types_linux.go b/vendor/github.com/cilium/ebpf/internal/unix/types_linux.go index 9363d0be81cff..86d2a10f96818 100644 --- a/vendor/github.com/cilium/ebpf/internal/unix/types_linux.go +++ b/vendor/github.com/cilium/ebpf/internal/unix/types_linux.go @@ -3,22 +3,29 @@ package unix import ( + "bytes" "syscall" linux "golang.org/x/sys/unix" ) const ( - ENOENT = linux.ENOENT - EEXIST = linux.EEXIST - EAGAIN = linux.EAGAIN - ENOSPC = linux.ENOSPC - EINVAL = linux.EINVAL - EPOLLIN = linux.EPOLLIN - EINTR = linux.EINTR - EPERM = linux.EPERM - ESRCH = linux.ESRCH - ENODEV = linux.ENODEV + ENOENT = linux.ENOENT + EEXIST = linux.EEXIST + EAGAIN = linux.EAGAIN + ENOSPC = linux.ENOSPC + EINVAL = linux.EINVAL + EPOLLIN = linux.EPOLLIN + EINTR = linux.EINTR + EPERM = linux.EPERM + ESRCH = linux.ESRCH + ENODEV = linux.ENODEV + // ENOTSUPP is not the same as ENOTSUP or EOPNOTSUP + ENOTSUPP = syscall.Errno(0x20c) + + EBADF = linux.EBADF + BPF_F_NO_PREALLOC = linux.BPF_F_NO_PREALLOC + BPF_F_NUMA_NODE = linux.BPF_F_NUMA_NODE BPF_F_RDONLY_PROG = linux.BPF_F_RDONLY_PROG BPF_F_WRONLY_PROG = linux.BPF_F_WRONLY_PROG BPF_OBJ_NAME_LEN = linux.BPF_OBJ_NAME_LEN @@ -39,6 +46,7 @@ const ( PERF_FLAG_FD_CLOEXEC = linux.PERF_FLAG_FD_CLOEXEC RLIM_INFINITY = linux.RLIM_INFINITY RLIMIT_MEMLOCK = linux.RLIMIT_MEMLOCK + BPF_STATS_RUN_TIME = linux.BPF_STATS_RUN_TIME ) // Statfs_t is a wrapper @@ -148,3 +156,15 @@ func Gettid() int { func Tgkill(tgid int, tid int, sig syscall.Signal) (err error) { return linux.Tgkill(tgid, tid, sig) } + +func KernelRelease() (string, error) { + var uname Utsname + err := Uname(&uname) + if err != nil { + return "", err + } + + end := bytes.IndexByte(uname.Release[:], 0) + release := string(uname.Release[:end]) + return release, nil +} diff --git a/vendor/github.com/cilium/ebpf/internal/unix/types_other.go b/vendor/github.com/cilium/ebpf/internal/unix/types_other.go index 2dea950f88816..8c291796a9b84 100644 --- a/vendor/github.com/cilium/ebpf/internal/unix/types_other.go +++ b/vendor/github.com/cilium/ebpf/internal/unix/types_other.go @@ -11,15 +11,21 @@ import ( var errNonLinux = fmt.Errorf("unsupported platform %s/%s", runtime.GOOS, runtime.GOARCH) const ( - ENOENT = syscall.ENOENT - EEXIST = syscall.EEXIST - EAGAIN = syscall.EAGAIN - ENOSPC = syscall.ENOSPC - EINVAL = syscall.EINVAL - EINTR = syscall.EINTR - EPERM = syscall.EPERM - ESRCH = syscall.ESRCH - ENODEV = syscall.ENODEV + ENOENT = syscall.ENOENT + EEXIST = syscall.EEXIST + EAGAIN = syscall.EAGAIN + ENOSPC = syscall.ENOSPC + EINVAL = syscall.EINVAL + EINTR = syscall.EINTR + EPERM = syscall.EPERM + ESRCH = syscall.ESRCH + ENODEV = syscall.ENODEV + EBADF = syscall.Errno(0) + // ENOTSUPP is not the same as ENOTSUP or EOPNOTSUP + ENOTSUPP = syscall.Errno(0x20c) + + BPF_F_NO_PREALLOC = 0 + BPF_F_NUMA_NODE = 0 BPF_F_RDONLY_PROG = 0 BPF_F_WRONLY_PROG = 0 BPF_OBJ_NAME_LEN = 0x10 @@ -41,6 +47,7 @@ const ( PERF_FLAG_FD_CLOEXEC = 0x8 RLIM_INFINITY = 0x7fffffffffffffff RLIMIT_MEMLOCK = 8 + BPF_STATS_RUN_TIME = 0 ) // Statfs_t is a wrapper @@ -215,3 +222,7 @@ func Gettid() int { func Tgkill(tgid int, tid int, sig syscall.Signal) (err error) { return errNonLinux } + +func KernelRelease() (string, error) { + return "", errNonLinux +} diff --git a/vendor/github.com/cilium/ebpf/link/cgroup.go b/vendor/github.com/cilium/ebpf/link/cgroup.go new file mode 100644 index 0000000000000..16a9439303611 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/cgroup.go @@ -0,0 +1,169 @@ +package link + +import ( + "errors" + "fmt" + "os" + + "github.com/cilium/ebpf" +) + +type cgroupAttachFlags uint32 + +// cgroup attach flags +const ( + flagAllowOverride cgroupAttachFlags = 1 << iota + flagAllowMulti + flagReplace +) + +type CgroupOptions struct { + // Path to a cgroupv2 folder. + Path string + // One of the AttachCgroup* constants + Attach ebpf.AttachType + // Program must be of type CGroup*, and the attach type must match Attach. + Program *ebpf.Program +} + +// AttachCgroup links a BPF program to a cgroup. +func AttachCgroup(opts CgroupOptions) (Link, error) { + cgroup, err := os.Open(opts.Path) + if err != nil { + return nil, fmt.Errorf("can't open cgroup: %s", err) + } + + clone, err := opts.Program.Clone() + if err != nil { + cgroup.Close() + return nil, err + } + + var cg Link + cg, err = newLinkCgroup(cgroup, opts.Attach, clone) + if errors.Is(err, ErrNotSupported) { + cg, err = newProgAttachCgroup(cgroup, opts.Attach, clone, flagAllowMulti) + } + if errors.Is(err, ErrNotSupported) { + cg, err = newProgAttachCgroup(cgroup, opts.Attach, clone, flagAllowOverride) + } + if err != nil { + cgroup.Close() + clone.Close() + return nil, err + } + + return cg, nil +} + +// LoadPinnedCgroup loads a pinned cgroup from a bpffs. +func LoadPinnedCgroup(fileName string) (Link, error) { + link, err := LoadPinnedRawLink(fileName) + if err != nil { + return nil, err + } + + return &linkCgroup{link}, nil +} + +type progAttachCgroup struct { + cgroup *os.File + current *ebpf.Program + attachType ebpf.AttachType + flags cgroupAttachFlags +} + +var _ Link = (*progAttachCgroup)(nil) + +func (cg *progAttachCgroup) isLink() {} + +func newProgAttachCgroup(cgroup *os.File, attach ebpf.AttachType, prog *ebpf.Program, flags cgroupAttachFlags) (*progAttachCgroup, error) { + if flags&flagAllowMulti > 0 { + if err := haveProgAttachReplace(); err != nil { + return nil, fmt.Errorf("can't support multiple programs: %w", err) + } + } + + err := RawAttachProgram(RawAttachProgramOptions{ + Target: int(cgroup.Fd()), + Program: prog, + Flags: uint32(flags), + Attach: attach, + }) + if err != nil { + return nil, fmt.Errorf("cgroup: %w", err) + } + + return &progAttachCgroup{cgroup, prog, attach, flags}, nil +} + +func (cg *progAttachCgroup) Close() error { + defer cg.cgroup.Close() + defer cg.current.Close() + + err := RawDetachProgram(RawDetachProgramOptions{ + Target: int(cg.cgroup.Fd()), + Program: cg.current, + Attach: cg.attachType, + }) + if err != nil { + return fmt.Errorf("close cgroup: %s", err) + } + return nil +} + +func (cg *progAttachCgroup) Update(prog *ebpf.Program) error { + new, err := prog.Clone() + if err != nil { + return err + } + + args := RawAttachProgramOptions{ + Target: int(cg.cgroup.Fd()), + Program: prog, + Attach: cg.attachType, + Flags: uint32(cg.flags), + } + + if cg.flags&flagAllowMulti > 0 { + // Atomically replacing multiple programs requires at least + // 5.5 (commit 7dd68b3279f17921 "bpf: Support replacing cgroup-bpf + // program in MULTI mode") + args.Flags |= uint32(flagReplace) + args.Replace = cg.current + } + + if err := RawAttachProgram(args); err != nil { + new.Close() + return fmt.Errorf("can't update cgroup: %s", err) + } + + cg.current.Close() + cg.current = new + return nil +} + +func (cg *progAttachCgroup) Pin(string) error { + return fmt.Errorf("can't pin cgroup: %w", ErrNotSupported) +} + +type linkCgroup struct { + *RawLink +} + +var _ Link = (*linkCgroup)(nil) + +func (cg *linkCgroup) isLink() {} + +func newLinkCgroup(cgroup *os.File, attach ebpf.AttachType, prog *ebpf.Program) (*linkCgroup, error) { + link, err := AttachRawLink(RawLinkOptions{ + Target: int(cgroup.Fd()), + Program: prog, + Attach: attach, + }) + if err != nil { + return nil, err + } + + return &linkCgroup{link}, err +} diff --git a/vendor/github.com/cilium/ebpf/link/doc.go b/vendor/github.com/cilium/ebpf/link/doc.go new file mode 100644 index 0000000000000..2bde35ed7a265 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/doc.go @@ -0,0 +1,2 @@ +// Package link allows attaching eBPF programs to various kernel hooks. +package link diff --git a/vendor/github.com/cilium/ebpf/link/iter.go b/vendor/github.com/cilium/ebpf/link/iter.go new file mode 100644 index 0000000000000..2b5f2846a9870 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/iter.go @@ -0,0 +1,91 @@ +package link + +import ( + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +type IterOptions struct { + // Program must be of type Tracing with attach type + // AttachTraceIter. The kind of iterator to attach to is + // determined at load time via the AttachTo field. + // + // AttachTo requires the kernel to include BTF of itself, + // and it to be compiled with a recent pahole (>= 1.16). + Program *ebpf.Program +} + +// AttachIter attaches a BPF seq_file iterator. +func AttachIter(opts IterOptions) (*Iter, error) { + link, err := AttachRawLink(RawLinkOptions{ + Program: opts.Program, + Attach: ebpf.AttachTraceIter, + }) + if err != nil { + return nil, fmt.Errorf("can't link iterator: %w", err) + } + + return &Iter{link}, err +} + +// LoadPinnedIter loads a pinned iterator from a bpffs. +func LoadPinnedIter(fileName string) (*Iter, error) { + link, err := LoadPinnedRawLink(fileName) + if err != nil { + return nil, err + } + + return &Iter{link}, err +} + +// Iter represents an attached bpf_iter. +type Iter struct { + link *RawLink +} + +var _ Link = (*Iter)(nil) + +func (it *Iter) isLink() {} + +// FD returns the underlying file descriptor. +func (it *Iter) FD() int { + return it.link.FD() +} + +// Close implements Link. +func (it *Iter) Close() error { + return it.link.Close() +} + +// Pin implements Link. +func (it *Iter) Pin(fileName string) error { + return it.link.Pin(fileName) +} + +// Update implements Link. +func (it *Iter) Update(new *ebpf.Program) error { + return it.link.Update(new) +} + +// Open creates a new instance of the iterator. +// +// Reading from the returned reader triggers the BPF program. +func (it *Iter) Open() (io.ReadCloser, error) { + linkFd, err := it.link.fd.Value() + if err != nil { + return nil, err + } + + attr := &bpfIterCreateAttr{ + linkFd: linkFd, + } + + fd, err := bpfIterCreate(attr) + if err != nil { + return nil, fmt.Errorf("can't create iterator: %w", err) + } + + return fd.File("bpf_iter"), nil +} diff --git a/vendor/github.com/cilium/ebpf/link/link.go b/vendor/github.com/cilium/ebpf/link/link.go new file mode 100644 index 0000000000000..48f1a552975d9 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/link.go @@ -0,0 +1,214 @@ +package link + +import ( + "fmt" + "unsafe" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal" +) + +var ErrNotSupported = internal.ErrNotSupported + +// Link represents a Program attached to a BPF hook. +type Link interface { + // Replace the current program with a new program. + // + // Passing a nil program is an error. May return an error wrapping ErrNotSupported. + Update(*ebpf.Program) error + + // Persist a link by pinning it into a bpffs. + // + // May return an error wrapping ErrNotSupported. + Pin(string) error + + // Close frees resources. + // + // The link will be broken unless it has been pinned. A link + // may continue past the lifetime of the process if Close is + // not called. + Close() error + + // Prevent external users from implementing this interface. + isLink() +} + +// ID uniquely identifies a BPF link. +type ID uint32 + +// RawLinkOptions control the creation of a raw link. +type RawLinkOptions struct { + // File descriptor to attach to. This differs for each attach type. + Target int + // Program to attach. + Program *ebpf.Program + // Attach must match the attach type of Program. + Attach ebpf.AttachType +} + +// RawLinkInfo contains metadata on a link. +type RawLinkInfo struct { + Type Type + ID ID + Program ebpf.ProgramID +} + +// RawLink is the low-level API to bpf_link. +// +// You should consider using the higher level interfaces in this +// package instead. +type RawLink struct { + fd *internal.FD +} + +// AttachRawLink creates a raw link. +func AttachRawLink(opts RawLinkOptions) (*RawLink, error) { + if err := haveBPFLink(); err != nil { + return nil, err + } + + if opts.Target < 0 { + return nil, fmt.Errorf("invalid target: %s", internal.ErrClosedFd) + } + + progFd := opts.Program.FD() + if progFd < 0 { + return nil, fmt.Errorf("invalid program: %s", internal.ErrClosedFd) + } + + attr := bpfLinkCreateAttr{ + targetFd: uint32(opts.Target), + progFd: uint32(progFd), + attachType: opts.Attach, + } + fd, err := bpfLinkCreate(&attr) + if err != nil { + return nil, fmt.Errorf("can't create link: %s", err) + } + + return &RawLink{fd}, nil +} + +// LoadPinnedRawLink loads a persisted link from a bpffs. +func LoadPinnedRawLink(fileName string) (*RawLink, error) { + return loadPinnedRawLink(fileName, UnspecifiedType) +} + +func loadPinnedRawLink(fileName string, typ Type) (*RawLink, error) { + fd, err := internal.BPFObjGet(fileName) + if err != nil { + return nil, fmt.Errorf("load pinned link: %s", err) + } + + link := &RawLink{fd} + if typ == UnspecifiedType { + return link, nil + } + + info, err := link.Info() + if err != nil { + link.Close() + return nil, fmt.Errorf("get pinned link info: %s", err) + } + + if info.Type != typ { + link.Close() + return nil, fmt.Errorf("link type %v doesn't match %v", info.Type, typ) + } + + return link, nil +} + +func (l *RawLink) isLink() {} + +// FD returns the raw file descriptor. +func (l *RawLink) FD() int { + fd, err := l.fd.Value() + if err != nil { + return -1 + } + return int(fd) +} + +// Close breaks the link. +// +// Use Pin if you want to make the link persistent. +func (l *RawLink) Close() error { + return l.fd.Close() +} + +// Pin persists a link past the lifetime of the process. +// +// Calling Close on a pinned Link will not break the link +// until the pin is removed. +func (l *RawLink) Pin(fileName string) error { + if err := internal.BPFObjPin(fileName, l.fd); err != nil { + return fmt.Errorf("can't pin link: %s", err) + } + return nil +} + +// Update implements Link. +func (l *RawLink) Update(new *ebpf.Program) error { + return l.UpdateArgs(RawLinkUpdateOptions{ + New: new, + }) +} + +// RawLinkUpdateOptions control the behaviour of RawLink.UpdateArgs. +type RawLinkUpdateOptions struct { + New *ebpf.Program + Old *ebpf.Program + Flags uint32 +} + +// UpdateArgs updates a link based on args. +func (l *RawLink) UpdateArgs(opts RawLinkUpdateOptions) error { + newFd := opts.New.FD() + if newFd < 0 { + return fmt.Errorf("invalid program: %s", internal.ErrClosedFd) + } + + var oldFd int + if opts.Old != nil { + oldFd = opts.Old.FD() + if oldFd < 0 { + return fmt.Errorf("invalid replacement program: %s", internal.ErrClosedFd) + } + } + + linkFd, err := l.fd.Value() + if err != nil { + return fmt.Errorf("can't update link: %s", err) + } + + attr := bpfLinkUpdateAttr{ + linkFd: linkFd, + newProgFd: uint32(newFd), + oldProgFd: uint32(oldFd), + flags: opts.Flags, + } + return bpfLinkUpdate(&attr) +} + +// struct bpf_link_info +type bpfLinkInfo struct { + typ uint32 + id uint32 + prog_id uint32 +} + +// Info returns metadata about the link. +func (l *RawLink) Info() (*RawLinkInfo, error) { + var info bpfLinkInfo + err := internal.BPFObjGetInfoByFD(l.fd, unsafe.Pointer(&info), unsafe.Sizeof(info)) + if err != nil { + return nil, fmt.Errorf("link info: %s", err) + } + + return &RawLinkInfo{ + Type(info.typ), + ID(info.id), + ebpf.ProgramID(info.prog_id), + }, nil +} diff --git a/vendor/github.com/cilium/ebpf/link/netns.go b/vendor/github.com/cilium/ebpf/link/netns.go new file mode 100644 index 0000000000000..3533ff0fa61b7 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/netns.go @@ -0,0 +1,60 @@ +package link + +import ( + "fmt" + + "github.com/cilium/ebpf" +) + +// NetNsInfo contains metadata about a network namespace link. +type NetNsInfo struct { + RawLinkInfo +} + +// NetNsLink is a program attached to a network namespace. +type NetNsLink struct { + *RawLink +} + +// AttachNetNs attaches a program to a network namespace. +func AttachNetNs(ns int, prog *ebpf.Program) (*NetNsLink, error) { + var attach ebpf.AttachType + switch t := prog.Type(); t { + case ebpf.FlowDissector: + attach = ebpf.AttachFlowDissector + case ebpf.SkLookup: + attach = ebpf.AttachSkLookup + default: + return nil, fmt.Errorf("can't attach %v to network namespace", t) + } + + link, err := AttachRawLink(RawLinkOptions{ + Target: ns, + Program: prog, + Attach: attach, + }) + if err != nil { + return nil, err + } + + return &NetNsLink{link}, nil +} + +// LoadPinnedNetNs loads a network namespace link from bpffs. +func LoadPinnedNetNs(fileName string) (*NetNsLink, error) { + link, err := loadPinnedRawLink(fileName, NetNsType) + if err != nil { + return nil, err + } + + return &NetNsLink{link}, nil +} + +// Info returns information about the link. +func (nns *NetNsLink) Info() (*NetNsInfo, error) { + info, err := nns.RawLink.Info() + if err != nil { + return nil, err + } + return &NetNsInfo{*info}, nil +} diff --git a/vendor/github.com/cilium/ebpf/link/program.go b/vendor/github.com/cilium/ebpf/link/program.go new file mode 100644 index 0000000000000..0fe9d37c4f896 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/program.go @@ -0,0 +1,76 @@ +package link + +import ( + "fmt" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal" +) + +type RawAttachProgramOptions struct { + // File descriptor to attach to. This differs for each attach type. + Target int + // Program to attach. + Program *ebpf.Program + // Program to replace (cgroups). + Replace *ebpf.Program + // Attach must match the attach type of Program (and Replace). + Attach ebpf.AttachType + // Flags control the attach behaviour. This differs for each attach type. + Flags uint32 +} + +// RawAttachProgram is a low level wrapper around BPF_PROG_ATTACH. +// +// You should use one of the higher level abstractions available in this +// package if possible. +func RawAttachProgram(opts RawAttachProgramOptions) error { + if err := haveProgAttach(); err != nil { + return err + } + + var replaceFd uint32 + if opts.Replace != nil { + replaceFd = uint32(opts.Replace.FD()) + } + + attr := internal.BPFProgAttachAttr{ + TargetFd: uint32(opts.Target), + AttachBpfFd: uint32(opts.Program.FD()), + ReplaceBpfFd: replaceFd, + AttachType: uint32(opts.Attach), + AttachFlags: uint32(opts.Flags), + } + + if err := internal.BPFProgAttach(&attr); err != nil { + return fmt.Errorf("can't attach program: %s", err) + } + return nil +} + +type RawDetachProgramOptions struct { + Target int + Program *ebpf.Program + Attach ebpf.AttachType +} + +// RawDetachProgram is a low level wrapper around BPF_PROG_DETACH. +// +// You should use one of the higher level abstractions available in this +// package if possible. +func RawDetachProgram(opts RawDetachProgramOptions) error { + if err := haveProgAttach(); err != nil { + return err + } + + attr := internal.BPFProgDetachAttr{ + TargetFd: uint32(opts.Target), + AttachBpfFd: uint32(opts.Program.FD()), + AttachType: uint32(opts.Attach), + } + if err := internal.BPFProgDetach(&attr); err != nil { + return fmt.Errorf("can't detach program: %s", err) + } + + return nil +} diff --git a/vendor/github.com/cilium/ebpf/link/raw_tracepoint.go b/vendor/github.com/cilium/ebpf/link/raw_tracepoint.go new file mode 100644 index 0000000000000..65652486f1cf1 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/raw_tracepoint.go @@ -0,0 +1,57 @@ +package link + +import ( + "fmt" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal" +) + +type RawTracepointOptions struct { + // Tracepoint name. + Name string + // Program must be of type RawTracepoint* + Program *ebpf.Program +} + +// AttachRawTracepoint links a BPF program to a raw_tracepoint. +// +// Requires at least Linux 4.17. +func AttachRawTracepoint(opts RawTracepointOptions) (Link, error) { + if t := opts.Program.Type(); t != ebpf.RawTracepoint && t != ebpf.RawTracepointWritable { + return nil, fmt.Errorf("invalid program type %s, expected RawTracepoint(Writable)", t) + } + if opts.Program.FD() < 0 { + return nil, fmt.Errorf("invalid program: %w", internal.ErrClosedFd) + } + + fd, err := bpfRawTracepointOpen(&bpfRawTracepointOpenAttr{ + name: internal.NewStringPointer(opts.Name), + fd: uint32(opts.Program.FD()), + }) + if err != nil { + return nil, err + } + + return &progAttachRawTracepoint{fd: fd}, nil +} + +type progAttachRawTracepoint struct { + fd *internal.FD +} + +var _ Link = (*progAttachRawTracepoint)(nil) + +func (rt *progAttachRawTracepoint) isLink() {} + +func (rt *progAttachRawTracepoint) Close() error { + return rt.fd.Close() +} + +func (rt *progAttachRawTracepoint) Update(_ *ebpf.Program) error { + return fmt.Errorf("can't update raw_tracepoint: %w", ErrNotSupported) +} + +func (rt *progAttachRawTracepoint) Pin(_ string) error { + return fmt.Errorf("can't pin raw_tracepoint: %w", ErrNotSupported) +} diff --git a/vendor/github.com/cilium/ebpf/link/syscalls.go b/vendor/github.com/cilium/ebpf/link/syscalls.go new file mode 100644 index 0000000000000..19326c8af8001 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/link/syscalls.go @@ -0,0 +1,173 @@ +package link + +import ( + "errors" + "unsafe" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/asm" + "github.com/cilium/ebpf/internal" + "github.com/cilium/ebpf/internal/unix" +) + +// Type is the kind of link. +type Type uint32 + +// Valid link types. +// +// Equivalent to enum bpf_link_type. +const ( + UnspecifiedType Type = iota + RawTracepointType + TracingType + CgroupType + IterType + NetNsType + XDPType +) + +var haveProgAttach = internal.FeatureTest("BPF_PROG_ATTACH", "4.10", func() error { + prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ + Type: ebpf.CGroupSKB, + AttachType: ebpf.AttachCGroupInetIngress, + License: "MIT", + Instructions: asm.Instructions{ + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + }, + }) + if err != nil { + return internal.ErrNotSupported + } + + // BPF_PROG_ATTACH was introduced at the same time as CGgroupSKB, + // so being able to load the program is enough to infer that we + // have the syscall. + prog.Close() + return nil +}) + +var haveProgAttachReplace = internal.FeatureTest("BPF_PROG_ATTACH atomic replacement", "5.5", func() error { + if err := haveProgAttach(); err != nil { + return err + } + + prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ + Type: ebpf.CGroupSKB, + AttachType: ebpf.AttachCGroupInetIngress, + License: "MIT", + Instructions: asm.Instructions{ + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + }, + }) + if err != nil { + return internal.ErrNotSupported + } + defer prog.Close() + + // We know that we have BPF_PROG_ATTACH since we can load CGroupSKB programs. + // If passing BPF_F_REPLACE gives us EINVAL we know that the feature isn't + // present. + attr := internal.BPFProgAttachAttr{ + // We rely on this being checked after attachFlags. + TargetFd: ^uint32(0), + AttachBpfFd: uint32(prog.FD()), + AttachType: uint32(ebpf.AttachCGroupInetIngress), + AttachFlags: uint32(flagReplace), + } + + err = internal.BPFProgAttach(&attr) + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported + } + if errors.Is(err, unix.EBADF) { + return nil + } + return err +}) + +type bpfLinkCreateAttr struct { + progFd uint32 + targetFd uint32 + attachType ebpf.AttachType + flags uint32 +} + +func bpfLinkCreate(attr *bpfLinkCreateAttr) (*internal.FD, error) { + ptr, err := internal.BPF(internal.BPF_LINK_CREATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + if err != nil { + return nil, err + } + return internal.NewFD(uint32(ptr)), nil +} + +type bpfLinkUpdateAttr struct { + linkFd uint32 + newProgFd uint32 + flags uint32 + oldProgFd uint32 +} + +func bpfLinkUpdate(attr *bpfLinkUpdateAttr) error { + _, err := internal.BPF(internal.BPF_LINK_UPDATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + return err +} + +var haveBPFLink = internal.FeatureTest("bpf_link", "5.7", func() error { + prog, err := ebpf.NewProgram(&ebpf.ProgramSpec{ + Type: ebpf.CGroupSKB, + AttachType: ebpf.AttachCGroupInetIngress, + License: "MIT", + Instructions: asm.Instructions{ + asm.Mov.Imm(asm.R0, 0), + asm.Return(), + }, + }) + if err != nil { + return internal.ErrNotSupported + } + defer prog.Close() + + attr := bpfLinkCreateAttr{ + // This is a hopefully invalid file descriptor, which triggers EBADF. + targetFd: ^uint32(0), + progFd: uint32(prog.FD()), + attachType: ebpf.AttachCGroupInetIngress, + } + _, err = bpfLinkCreate(&attr) + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported + } + if errors.Is(err, unix.EBADF) { + return nil + } + return err +}) + +type bpfIterCreateAttr struct { + linkFd uint32 + flags uint32 +} + +func bpfIterCreate(attr *bpfIterCreateAttr) (*internal.FD, error) { + ptr, err := internal.BPF(internal.BPF_ITER_CREATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + if err == nil { + return internal.NewFD(uint32(ptr)), nil + } + return nil, err +} + +type bpfRawTracepointOpenAttr struct { + name internal.Pointer + fd uint32 + _ uint32 +} + +func bpfRawTracepointOpen(attr *bpfRawTracepointOpenAttr) (*internal.FD, error) { + ptr, err := internal.BPF(internal.BPF_RAW_TRACEPOINT_OPEN, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) + if err == nil { + return internal.NewFD(uint32(ptr)), nil + } + return nil, err +} diff --git a/vendor/github.com/cilium/ebpf/linker.go b/vendor/github.com/cilium/ebpf/linker.go index 1bb8f61c25aa0..f843bb25e7ba2 100644 --- a/vendor/github.com/cilium/ebpf/linker.go +++ b/vendor/github.com/cilium/ebpf/linker.go @@ -84,3 +84,50 @@ func needSection(insns, section asm.Instructions) (bool, error) { // None of the functions in the section are called. return false, nil } + +func fixupJumpsAndCalls(insns asm.Instructions) error { + symbolOffsets := make(map[string]asm.RawInstructionOffset) + iter := insns.Iterate() + for iter.Next() { + ins := iter.Ins + + if ins.Symbol == "" { + continue + } + + if _, ok := symbolOffsets[ins.Symbol]; ok { + return fmt.Errorf("duplicate symbol %s", ins.Symbol) + } + + symbolOffsets[ins.Symbol] = iter.Offset + } + + iter = insns.Iterate() + for iter.Next() { + i := iter.Index + offset := iter.Offset + ins := iter.Ins + + switch { + case ins.IsFunctionCall() && ins.Constant == -1: + // Rewrite bpf to bpf call + callOffset, ok := symbolOffsets[ins.Reference] + if !ok { + return fmt.Errorf("instruction %d: reference to missing symbol %q", i, ins.Reference) + } + + ins.Constant = int64(callOffset - offset - 1) + + case ins.OpCode.Class() == asm.JumpClass && ins.Offset == -1: + // Rewrite jump to label + jumpOffset, ok := symbolOffsets[ins.Reference] + if !ok { + return fmt.Errorf("instruction %d: reference to missing symbol %q", i, ins.Reference) + } + + ins.Offset = int16(jumpOffset - offset - 1) + } + } + + return nil +} diff --git a/vendor/github.com/cilium/ebpf/map.go b/vendor/github.com/cilium/ebpf/map.go index 461b995a54bb1..316fc37b1c237 100644 --- a/vendor/github.com/cilium/ebpf/map.go +++ b/vendor/github.com/cilium/ebpf/map.go @@ -3,6 +3,9 @@ package ebpf import ( "errors" "fmt" + "io" + "path/filepath" + "reflect" "strings" "github.com/cilium/ebpf/internal" @@ -17,6 +20,14 @@ var ( ErrIterationAborted = errors.New("iteration aborted") ) +// MapOptions control loading a map into the kernel. +type MapOptions struct { + // The base path to pin maps in if requested via PinByName. + // Existing maps will be re-used if they are compatible, otherwise an + // error is returned. + PinPath string +} + // MapID represents the unique ID of an eBPF map type MapID uint32 @@ -31,6 +42,15 @@ type MapSpec struct { MaxEntries uint32 Flags uint32 + // Automatically pin and load a map from MapOptions.PinPath. + // Generates an error if an existing pinned map is incompatible with the MapSpec. + Pinning PinType + + // Specify numa node during map creation + // (effective only if unix.BPF_F_NUMA_NODE flag is set, + // which can be imported from golang.org/x/sys/unix) + NumaNode uint32 + // The initial contents of the map. May be nil. Contents []MapKV @@ -69,6 +89,26 @@ type MapKV struct { Value interface{} } +func (ms *MapSpec) checkCompatibility(m *Map) error { + switch { + case m.typ != ms.Type: + return fmt.Errorf("expected type %v, got %v", ms.Type, m.typ) + + case m.keySize != ms.KeySize: + return fmt.Errorf("expected key size %v, got %v", ms.KeySize, m.keySize) + + case m.valueSize != ms.ValueSize: + return fmt.Errorf("expected value size %v, got %v", ms.ValueSize, m.valueSize) + + case m.maxEntries != ms.MaxEntries: + return fmt.Errorf("expected max entries %v, got %v", ms.MaxEntries, m.maxEntries) + + case m.flags != ms.Flags: + return fmt.Errorf("expected flags %v, got %v", ms.Flags, m.flags) + } + return nil +} + // Map represents a Map file descriptor. // // It is not safe to close a map which is used by other goroutines. @@ -79,9 +119,14 @@ type MapKV struct { // Implement encoding.BinaryMarshaler or encoding.BinaryUnmarshaler // if you require custom encoding. type Map struct { - name string - fd *internal.FD - abi MapABI + name string + fd *internal.FD + typ MapType + keySize uint32 + valueSize uint32 + maxEntries uint32 + flags uint32 + pinnedPath string // Per CPU maps return values larger than the size in the spec fullValueSize int } @@ -93,57 +138,113 @@ func NewMapFromFD(fd int) (*Map, error) { if fd < 0 { return nil, errors.New("invalid fd") } - bpfFd := internal.NewFD(uint32(fd)) - name, abi, err := newMapABIFromFd(bpfFd) + return newMapFromFD(internal.NewFD(uint32(fd))) +} + +func newMapFromFD(fd *internal.FD) (*Map, error) { + info, err := newMapInfoFromFd(fd) if err != nil { - bpfFd.Forget() - return nil, err + fd.Close() + return nil, fmt.Errorf("get map info: %s", err) } - return newMap(bpfFd, name, abi) + + return newMap(fd, info.Name, info.Type, info.KeySize, info.ValueSize, info.MaxEntries, info.Flags) } // NewMap creates a new Map. // +// It's equivalent to calling NewMapWithOptions with default options. +func NewMap(spec *MapSpec) (*Map, error) { + return NewMapWithOptions(spec, MapOptions{}) +} + +// NewMapWithOptions creates a new Map. +// // Creating a map for the first time will perform feature detection // by creating small, temporary maps. // // The caller is responsible for ensuring the process' rlimit is set // sufficiently high for locking memory during map creation. This can be done -// by calling unix.Setrlimit with unix.RLIMIT_MEMLOCK prior to calling NewMap. -func NewMap(spec *MapSpec) (*Map, error) { - if spec.BTF == nil { - return newMapWithBTF(spec, nil) - } - - handle, err := btf.NewHandle(btf.MapSpec(spec.BTF)) - if err != nil && !errors.Is(err, btf.ErrNotSupported) { - return nil, fmt.Errorf("can't load BTF: %w", err) - } +// by calling unix.Setrlimit with unix.RLIMIT_MEMLOCK prior to calling NewMapWithOptions. +func NewMapWithOptions(spec *MapSpec, opts MapOptions) (*Map, error) { + btfs := make(btfHandleCache) + defer btfs.close() - return newMapWithBTF(spec, handle) + return newMapWithOptions(spec, opts, btfs) } -func newMapWithBTF(spec *MapSpec, handle *btf.Handle) (*Map, error) { - if spec.Type != ArrayOfMaps && spec.Type != HashOfMaps { - return createMap(spec, nil, handle) +func newMapWithOptions(spec *MapSpec, opts MapOptions, btfs btfHandleCache) (*Map, error) { + switch spec.Pinning { + case PinByName: + if spec.Name == "" || opts.PinPath == "" { + return nil, fmt.Errorf("pin by name: missing Name or PinPath") + } + + m, err := LoadPinnedMap(filepath.Join(opts.PinPath, spec.Name)) + if errors.Is(err, unix.ENOENT) { + break + } + if err != nil { + return nil, fmt.Errorf("load pinned map: %s", err) + } + + if err := spec.checkCompatibility(m); err != nil { + m.Close() + return nil, fmt.Errorf("use pinned map %s: %s", spec.Name, err) + } + + return m, nil + + case PinNone: + // Nothing to do here + + default: + return nil, fmt.Errorf("unsupported pin type %d", int(spec.Pinning)) } - if spec.InnerMap == nil { - return nil, fmt.Errorf("%s requires InnerMap", spec.Type) + var innerFd *internal.FD + if spec.Type == ArrayOfMaps || spec.Type == HashOfMaps { + if spec.InnerMap == nil { + return nil, fmt.Errorf("%s requires InnerMap", spec.Type) + } + + if spec.InnerMap.Pinning != PinNone { + return nil, errors.New("inner maps cannot be pinned") + } + + template, err := createMap(spec.InnerMap, nil, opts, btfs) + if err != nil { + return nil, err + } + defer template.Close() + + innerFd = template.fd } - template, err := createMap(spec.InnerMap, nil, handle) + m, err := createMap(spec, innerFd, opts, btfs) if err != nil { return nil, err } - defer template.Close() - return createMap(spec, template.fd, handle) + if spec.Pinning == PinByName { + if err := m.Pin(filepath.Join(opts.PinPath, spec.Name)); err != nil { + m.Close() + return nil, fmt.Errorf("pin map: %s", err) + } + } + + return m, nil } -func createMap(spec *MapSpec, inner *internal.FD, handle *btf.Handle) (*Map, error) { - abi := newMapABIFromSpec(spec) +func createMap(spec *MapSpec, inner *internal.FD, opts MapOptions, btfs btfHandleCache) (_ *Map, err error) { + closeOnError := func(closer io.Closer) { + if err != nil { + closer.Close() + } + } + + spec = spec.Copy() switch spec.Type { case ArrayOfMaps: @@ -153,43 +254,44 @@ func createMap(spec *MapSpec, inner *internal.FD, handle *btf.Handle) (*Map, err return nil, err } - if abi.ValueSize != 0 && abi.ValueSize != 4 { + if spec.ValueSize != 0 && spec.ValueSize != 4 { return nil, errors.New("ValueSize must be zero or four for map of map") } - abi.ValueSize = 4 + spec.ValueSize = 4 case PerfEventArray: - if abi.KeySize != 0 && abi.KeySize != 4 { + if spec.KeySize != 0 && spec.KeySize != 4 { return nil, errors.New("KeySize must be zero or four for perf event array") } - abi.KeySize = 4 + spec.KeySize = 4 - if abi.ValueSize != 0 && abi.ValueSize != 4 { + if spec.ValueSize != 0 && spec.ValueSize != 4 { return nil, errors.New("ValueSize must be zero or four for perf event array") } - abi.ValueSize = 4 + spec.ValueSize = 4 - if abi.MaxEntries == 0 { + if spec.MaxEntries == 0 { n, err := internal.PossibleCPUs() if err != nil { return nil, fmt.Errorf("perf event array: %w", err) } - abi.MaxEntries = uint32(n) + spec.MaxEntries = uint32(n) } } - if abi.Flags&(unix.BPF_F_RDONLY_PROG|unix.BPF_F_WRONLY_PROG) > 0 || spec.Freeze { + if spec.Flags&(unix.BPF_F_RDONLY_PROG|unix.BPF_F_WRONLY_PROG) > 0 || spec.Freeze { if err := haveMapMutabilityModifiers(); err != nil { return nil, fmt.Errorf("map create: %w", err) } } attr := bpfMapCreateAttr{ - mapType: abi.Type, - keySize: abi.KeySize, - valueSize: abi.ValueSize, - maxEntries: abi.MaxEntries, - flags: abi.Flags, + mapType: spec.Type, + keySize: spec.KeySize, + valueSize: spec.ValueSize, + maxEntries: spec.MaxEntries, + flags: spec.Flags, + numaNode: spec.NumaNode, } if inner != nil { @@ -200,34 +302,48 @@ func createMap(spec *MapSpec, inner *internal.FD, handle *btf.Handle) (*Map, err } } - if handle != nil && spec.BTF != nil { - attr.btfFd = uint32(handle.FD()) - attr.btfKeyTypeID = btf.MapKey(spec.BTF).ID() - attr.btfValueTypeID = btf.MapValue(spec.BTF).ID() - } - if haveObjName() == nil { attr.mapName = newBPFObjName(spec.Name) } + var btfDisabled bool + if spec.BTF != nil { + handle, err := btfs.load(btf.MapSpec(spec.BTF)) + btfDisabled = errors.Is(err, btf.ErrNotSupported) + if err != nil && !btfDisabled { + return nil, fmt.Errorf("load BTF: %w", err) + } + + if handle != nil { + attr.btfFd = uint32(handle.FD()) + attr.btfKeyTypeID = btf.MapKey(spec.BTF).ID() + attr.btfValueTypeID = btf.MapValue(spec.BTF).ID() + } + } + fd, err := bpfMapCreate(&attr) if err != nil { + if errors.Is(err, unix.EPERM) { + return nil, fmt.Errorf("map create: RLIMIT_MEMLOCK may be too low: %w", err) + } + if btfDisabled { + return nil, fmt.Errorf("map create without BTF: %w", err) + } return nil, fmt.Errorf("map create: %w", err) } + defer closeOnError(fd) - m, err := newMap(fd, spec.Name, abi) + m, err := newMap(fd, spec.Name, spec.Type, spec.KeySize, spec.ValueSize, spec.MaxEntries, spec.Flags) if err != nil { - return nil, err + return nil, fmt.Errorf("map create: %w", err) } if err := m.populate(spec.Contents); err != nil { - m.Close() return nil, fmt.Errorf("map create: can't set initial contents: %w", err) } if spec.Freeze { if err := m.Freeze(); err != nil { - m.Close() return nil, fmt.Errorf("can't freeze map: %w", err) } } @@ -235,15 +351,20 @@ func createMap(spec *MapSpec, inner *internal.FD, handle *btf.Handle) (*Map, err return m, nil } -func newMap(fd *internal.FD, name string, abi *MapABI) (*Map, error) { +func newMap(fd *internal.FD, name string, typ MapType, keySize, valueSize, maxEntries, flags uint32) (*Map, error) { m := &Map{ name, fd, - *abi, - int(abi.ValueSize), + typ, + keySize, + valueSize, + maxEntries, + flags, + "", + int(valueSize), } - if !abi.Type.hasPerCPUValue() { + if !typ.hasPerCPUValue() { return m, nil } @@ -252,20 +373,45 @@ func newMap(fd *internal.FD, name string, abi *MapABI) (*Map, error) { return nil, err } - m.fullValueSize = align(int(abi.ValueSize), 8) * possibleCPUs + m.fullValueSize = align(int(valueSize), 8) * possibleCPUs return m, nil } func (m *Map) String() string { if m.name != "" { - return fmt.Sprintf("%s(%s)#%v", m.abi.Type, m.name, m.fd) + return fmt.Sprintf("%s(%s)#%v", m.typ, m.name, m.fd) } - return fmt.Sprintf("%s#%v", m.abi.Type, m.fd) + return fmt.Sprintf("%s#%v", m.typ, m.fd) +} + +// Type returns the underlying type of the map. +func (m *Map) Type() MapType { + return m.typ +} + +// KeySize returns the size of the map key in bytes. +func (m *Map) KeySize() uint32 { + return m.keySize +} + +// ValueSize returns the size of the map value in bytes. +func (m *Map) ValueSize() uint32 { + return m.valueSize +} + +// MaxEntries returns the maximum number of elements the map can hold. +func (m *Map) MaxEntries() uint32 { + return m.maxEntries +} + +// Flags returns the flags of the map. +func (m *Map) Flags() uint32 { + return m.flags } -// ABI gets the ABI of the Map -func (m *Map) ABI() MapABI { - return m.abi +// Info returns metadata about the map. +func (m *Map) Info() (*MapInfo, error) { + return newMapInfoFromFd(m.fd) } // Lookup retrieves a value from a Map. @@ -273,54 +419,14 @@ func (m *Map) ABI() MapABI { // Calls Close() on valueOut if it is of type **Map or **Program, // and *valueOut is not nil. // -// Returns an error if the key doesn't exist, see IsNotExist. +// Returns an error if the key doesn't exist, see ErrKeyNotExist. func (m *Map) Lookup(key, valueOut interface{}) error { valuePtr, valueBytes := makeBuffer(valueOut, m.fullValueSize) - if err := m.lookup(key, valuePtr); err != nil { return err } - if valueBytes == nil { - return nil - } - - if m.abi.Type.hasPerCPUValue() { - return unmarshalPerCPUValue(valueOut, int(m.abi.ValueSize), valueBytes) - } - - switch value := valueOut.(type) { - case **Map: - m, err := unmarshalMap(valueBytes) - if err != nil { - return err - } - - (*value).Close() - *value = m - return nil - case *Map: - return fmt.Errorf("can't unmarshal into %T, need %T", value, (**Map)(nil)) - case Map: - return fmt.Errorf("can't unmarshal into %T, need %T", value, (**Map)(nil)) - - case **Program: - p, err := unmarshalProgram(valueBytes) - if err != nil { - return err - } - - (*value).Close() - *value = p - return nil - case *Program: - return fmt.Errorf("can't unmarshal into %T, need %T", value, (**Program)(nil)) - case Program: - return fmt.Errorf("can't unmarshal into %T, need %T", value, (**Program)(nil)) - - default: - return unmarshalBytes(valueOut, valueBytes) - } + return m.unmarshalValue(valueOut, valueBytes) } // LookupAndDelete retrieves and deletes a value from a Map. @@ -329,7 +435,7 @@ func (m *Map) Lookup(key, valueOut interface{}) error { func (m *Map) LookupAndDelete(key, valueOut interface{}) error { valuePtr, valueBytes := makeBuffer(valueOut, m.fullValueSize) - keyPtr, err := marshalPtr(key, int(m.abi.KeySize)) + keyPtr, err := m.marshalKey(key) if err != nil { return fmt.Errorf("can't marshal key: %w", err) } @@ -338,7 +444,7 @@ func (m *Map) LookupAndDelete(key, valueOut interface{}) error { return fmt.Errorf("lookup and delete failed: %w", err) } - return unmarshalBytes(valueOut, valueBytes) + return m.unmarshalValue(valueOut, valueBytes) } // LookupBytes gets a value from Map. @@ -357,7 +463,7 @@ func (m *Map) LookupBytes(key interface{}) ([]byte, error) { } func (m *Map) lookup(key interface{}, valueOut internal.Pointer) error { - keyPtr, err := marshalPtr(key, int(m.abi.KeySize)) + keyPtr, err := m.marshalKey(key) if err != nil { return fmt.Errorf("can't marshal key: %w", err) } @@ -391,17 +497,12 @@ func (m *Map) Put(key, value interface{}) error { // Update changes the value of a key. func (m *Map) Update(key, value interface{}, flags MapUpdateFlags) error { - keyPtr, err := marshalPtr(key, int(m.abi.KeySize)) + keyPtr, err := m.marshalKey(key) if err != nil { return fmt.Errorf("can't marshal key: %w", err) } - var valuePtr internal.Pointer - if m.abi.Type.hasPerCPUValue() { - valuePtr, err = marshalPerCPUValue(value, int(m.abi.ValueSize)) - } else { - valuePtr, err = marshalPtr(value, int(m.abi.ValueSize)) - } + valuePtr, err := m.marshalValue(value) if err != nil { return fmt.Errorf("can't marshal value: %w", err) } @@ -417,7 +518,7 @@ func (m *Map) Update(key, value interface{}, flags MapUpdateFlags) error { // // Returns ErrKeyNotExist if the key does not exist. func (m *Map) Delete(key interface{}) error { - keyPtr, err := marshalPtr(key, int(m.abi.KeySize)) + keyPtr, err := m.marshalKey(key) if err != nil { return fmt.Errorf("can't marshal key: %w", err) } @@ -434,17 +535,13 @@ func (m *Map) Delete(key interface{}) error { // // Returns ErrKeyNotExist if there is no next key. func (m *Map) NextKey(key, nextKeyOut interface{}) error { - nextKeyPtr, nextKeyBytes := makeBuffer(nextKeyOut, int(m.abi.KeySize)) + nextKeyPtr, nextKeyBytes := makeBuffer(nextKeyOut, int(m.keySize)) if err := m.nextKey(key, nextKeyPtr); err != nil { return err } - if nextKeyBytes == nil { - return nil - } - - if err := unmarshalBytes(nextKeyOut, nextKeyBytes); err != nil { + if err := m.unmarshalKey(nextKeyOut, nextKeyBytes); err != nil { return fmt.Errorf("can't unmarshal next key: %w", err) } return nil @@ -458,7 +555,7 @@ func (m *Map) NextKey(key, nextKeyOut interface{}) error { // // Returns nil if there are no more keys. func (m *Map) NextKeyBytes(key interface{}) ([]byte, error) { - nextKey := make([]byte, m.abi.KeySize) + nextKey := make([]byte, m.keySize) nextKeyPtr := internal.NewSlicePointer(nextKey) err := m.nextKey(key, nextKeyPtr) @@ -476,7 +573,7 @@ func (m *Map) nextKey(key interface{}, nextKeyOut internal.Pointer) error { ) if key != nil { - keyPtr, err = marshalPtr(key, int(m.abi.KeySize)) + keyPtr, err = m.marshalKey(key) if err != nil { return fmt.Errorf("can't marshal key: %w", err) } @@ -488,6 +585,158 @@ func (m *Map) nextKey(key interface{}, nextKeyOut internal.Pointer) error { return nil } +// BatchLookup looks up many elements in a map at once. +// +// "keysOut" and "valuesOut" must be of type slice, a pointer +// to a slice or buffer will not work. +// "prevKey" is the key to start the batch lookup from, it will +// *not* be included in the results. Use nil to start at the first key. +// +// ErrKeyNotExist is returned when the batch lookup has reached +// the end of all possible results, even when partial results +// are returned. It should be used to evaluate when lookup is "done". +func (m *Map) BatchLookup(prevKey, nextKeyOut, keysOut, valuesOut interface{}, opts *BatchOptions) (int, error) { + return m.batchLookup(internal.BPF_MAP_LOOKUP_BATCH, prevKey, nextKeyOut, keysOut, valuesOut, opts) +} + +// BatchLookupAndDelete looks up many elements in a map at once, +// +// It then deletes all those elements. +// "keysOut" and "valuesOut" must be of type slice, a pointer +// to a slice or buffer will not work. +// "prevKey" is the key to start the batch lookup from, it will +// *not* be included in the results. Use nil to start at the first key. +// +// ErrKeyNotExist is returned when the batch lookup has reached +// the end of all possible results, even when partial results +// are returned. It should be used to evaluate when lookup is "done". +func (m *Map) BatchLookupAndDelete(prevKey, nextKeyOut, keysOut, valuesOut interface{}, opts *BatchOptions) (int, error) { + return m.batchLookup(internal.BPF_MAP_LOOKUP_AND_DELETE_BATCH, prevKey, nextKeyOut, keysOut, valuesOut, opts) +} + +func (m *Map) batchLookup(cmd internal.BPFCmd, startKey, nextKeyOut, keysOut, valuesOut interface{}, opts *BatchOptions) (int, error) { + if err := haveBatchAPI(); err != nil { + return 0, err + } + if m.typ.hasPerCPUValue() { + return 0, ErrNotSupported + } + keysValue := reflect.ValueOf(keysOut) + if keysValue.Kind() != reflect.Slice { + return 0, fmt.Errorf("keys must be a slice") + } + valuesValue := reflect.ValueOf(valuesOut) + if valuesValue.Kind() != reflect.Slice { + return 0, fmt.Errorf("valuesOut must be a slice") + } + count := keysValue.Len() + if count != valuesValue.Len() { + return 0, fmt.Errorf("keysOut and valuesOut must be the same length") + } + keyBuf := make([]byte, count*int(m.keySize)) + keyPtr := internal.NewSlicePointer(keyBuf) + valueBuf := make([]byte, count*int(m.fullValueSize)) + valuePtr := internal.NewSlicePointer(valueBuf) + + var ( + startPtr internal.Pointer + err error + retErr error + ) + if startKey != nil { + startPtr, err = marshalPtr(startKey, int(m.keySize)) + if err != nil { + return 0, err + } + } + nextPtr, nextBuf := makeBuffer(nextKeyOut, int(m.keySize)) + + ct, err := bpfMapBatch(cmd, m.fd, startPtr, nextPtr, keyPtr, valuePtr, uint32(count), opts) + if err != nil { + if !errors.Is(err, ErrKeyNotExist) { + return 0, err + } + retErr = ErrKeyNotExist + } + + err = m.unmarshalKey(nextKeyOut, nextBuf) + if err != nil { + return 0, err + } + err = unmarshalBytes(keysOut, keyBuf) + if err != nil { + return 0, err + } + err = unmarshalBytes(valuesOut, valueBuf) + if err != nil { + retErr = err + } + return int(ct), retErr +} + +// BatchUpdate updates the map with multiple keys and values +// simultaneously. +// "keys" and "values" must be of type slice, a pointer +// to a slice or buffer will not work. +func (m *Map) BatchUpdate(keys, values interface{}, opts *BatchOptions) (int, error) { + if err := haveBatchAPI(); err != nil { + return 0, err + } + if m.typ.hasPerCPUValue() { + return 0, ErrNotSupported + } + keysValue := reflect.ValueOf(keys) + if keysValue.Kind() != reflect.Slice { + return 0, fmt.Errorf("keys must be a slice") + } + valuesValue := reflect.ValueOf(values) + if valuesValue.Kind() != reflect.Slice { + return 0, fmt.Errorf("values must be a slice") + } + var ( + count = keysValue.Len() + valuePtr internal.Pointer + err error + ) + if count != valuesValue.Len() { + return 0, fmt.Errorf("keys and values must be the same length") + } + keyPtr, err := marshalPtr(keys, count*int(m.keySize)) + if err != nil { + return 0, err + } + valuePtr, err = marshalPtr(values, count*int(m.valueSize)) + if err != nil { + return 0, err + } + var nilPtr internal.Pointer + ct, err := bpfMapBatch(internal.BPF_MAP_UPDATE_BATCH, m.fd, nilPtr, nilPtr, keyPtr, valuePtr, uint32(count), opts) + return int(ct), err +} + +// BatchDelete batch deletes entries in the map by keys. +// "keys" must be of type slice, a pointer to a slice or buffer will not work. +func (m *Map) BatchDelete(keys interface{}, opts *BatchOptions) (int, error) { + if err := haveBatchAPI(); err != nil { + return 0, err + } + if m.typ.hasPerCPUValue() { + return 0, ErrNotSupported + } + keysValue := reflect.ValueOf(keys) + if keysValue.Kind() != reflect.Slice { + return 0, fmt.Errorf("keys must be a slice") + } + count := keysValue.Len() + keyPtr, err := marshalPtr(keys, count*int(m.keySize)) + if err != nil { + return 0, fmt.Errorf("cannot marshal keys: %v", err) + } + var nilPtr internal.Pointer + ct, err := bpfMapBatch(internal.BPF_MAP_DELETE_BATCH, m.fd, nilPtr, nilPtr, keyPtr, nilPtr, uint32(count), opts) + return int(ct), err +} + // Iterate traverses a map. // // It's safe to create multiple iterators at the same time. @@ -527,6 +776,7 @@ func (m *Map) FD() int { // // Closing the duplicate does not affect the original, and vice versa. // Changes made to the map are reflected by both instances however. +// If the original map was pinned, the cloned map will not be pinned by default. // // Cloning a nil Map returns nil. func (m *Map) Clone() (*Map, error) { @@ -539,14 +789,53 @@ func (m *Map) Clone() (*Map, error) { return nil, fmt.Errorf("can't clone map: %w", err) } - return newMap(dup, m.name, &m.abi) + return &Map{ + m.name, + dup, + m.typ, + m.keySize, + m.valueSize, + m.maxEntries, + m.flags, + "", + m.fullValueSize, + }, nil } -// Pin persists the map past the lifetime of the process that created it. +// Pin persists the map on the BPF virtual file system past the lifetime of +// the process that created it . +// +// Calling Pin on a previously pinned map will override the path. +// You can Clone a map to pin it to a different path. // -// This requires bpffs to be mounted above fileName. See http://cilium.readthedocs.io/en/doc-1.0/kubernetes/install/#mounting-the-bpf-fs-optional +// This requires bpffs to be mounted above fileName. See https://docs.cilium.io/en/k8s-doc/admin/#admin-mount-bpffs func (m *Map) Pin(fileName string) error { - return internal.BPFObjPin(fileName, m.fd) + if err := pin(m.pinnedPath, fileName, m.fd); err != nil { + return err + } + m.pinnedPath = fileName + return nil +} + +// Unpin removes the persisted state for the map from the BPF virtual filesystem. +// +// Failed calls to Unpin will not alter the state returned by IsPinned. +// +// Unpinning an unpinned Map returns nil. +func (m *Map) Unpin() error { + if err := unpin(m.pinnedPath); err != nil { + return err + } + m.pinnedPath = "" + return nil +} + +// IsPinned returns true if the map has a non-empty pinned path. +func (m *Map) IsPinned() bool { + if m.pinnedPath == "" { + return false + } + return true } // Freeze prevents a map to be modified from user space. @@ -572,45 +861,147 @@ func (m *Map) populate(contents []MapKV) error { return nil } -// LoadPinnedMap load a Map from a BPF file. -// -// The function is not compatible with nested maps. -// Use LoadPinnedMapExplicit in these situations. -func LoadPinnedMap(fileName string) (*Map, error) { - fd, err := internal.BPFObjGet(fileName) - if err != nil { - return nil, err +func (m *Map) marshalKey(data interface{}) (internal.Pointer, error) { + if data == nil { + if m.keySize == 0 { + // Queues have a key length of zero, so passing nil here is valid. + return internal.NewPointer(nil), nil + } + return internal.Pointer{}, errors.New("can't use nil as key of map") } - name, abi, err := newMapABIFromFd(fd) + + return marshalPtr(data, int(m.keySize)) +} + +func (m *Map) unmarshalKey(data interface{}, buf []byte) error { + if buf == nil { + // This is from a makeBuffer call, nothing do do here. + return nil + } + + return unmarshalBytes(data, buf) +} + +func (m *Map) marshalValue(data interface{}) (internal.Pointer, error) { + if m.typ.hasPerCPUValue() { + return marshalPerCPUValue(data, int(m.valueSize)) + } + + var ( + buf []byte + err error + ) + + switch value := data.(type) { + case *Map: + if !m.typ.canStoreMap() { + return internal.Pointer{}, fmt.Errorf("can't store map in %s", m.typ) + } + buf, err = marshalMap(value, int(m.valueSize)) + + case *Program: + if !m.typ.canStoreProgram() { + return internal.Pointer{}, fmt.Errorf("can't store program in %s", m.typ) + } + buf, err = marshalProgram(value, int(m.valueSize)) + + default: + return marshalPtr(data, int(m.valueSize)) + } + if err != nil { - _ = fd.Close() - return nil, err + return internal.Pointer{}, err + } + + return internal.NewSlicePointer(buf), nil +} + +func (m *Map) unmarshalValue(value interface{}, buf []byte) error { + if buf == nil { + // This is from a makeBuffer call, nothing do do here. + return nil + } + + if m.typ.hasPerCPUValue() { + return unmarshalPerCPUValue(value, int(m.valueSize), buf) + } + + switch value := value.(type) { + case **Map: + if !m.typ.canStoreMap() { + return fmt.Errorf("can't read a map from %s", m.typ) + } + + other, err := unmarshalMap(buf) + if err != nil { + return err + } + + (*value).Close() + *value = other + return nil + + case *Map: + if !m.typ.canStoreMap() { + return fmt.Errorf("can't read a map from %s", m.typ) + } + return errors.New("require pointer to *Map") + + case **Program: + if !m.typ.canStoreProgram() { + return fmt.Errorf("can't read a program from %s", m.typ) + } + + other, err := unmarshalProgram(buf) + if err != nil { + return err + } + + (*value).Close() + *value = other + return nil + + case *Program: + if !m.typ.canStoreProgram() { + return fmt.Errorf("can't read a program from %s", m.typ) + } + return errors.New("require pointer to *Program") } - return newMap(fd, name, abi) + + return unmarshalBytes(value, buf) } -// LoadPinnedMapExplicit loads a map with explicit parameters. -func LoadPinnedMapExplicit(fileName string, abi *MapABI) (*Map, error) { +// LoadPinnedMap load a Map from a BPF file. +func LoadPinnedMap(fileName string) (*Map, error) { fd, err := internal.BPFObjGet(fileName) if err != nil { return nil, err } - return newMap(fd, "", abi) + + m, err := newMapFromFD(fd) + if err == nil { + m.pinnedPath = fileName + } + + return m, err } +// unmarshalMap creates a map from a map ID encoded in host endianness. func unmarshalMap(buf []byte) (*Map, error) { if len(buf) != 4 { return nil, errors.New("map id requires 4 byte value") } - // Looking up an entry in a nested map or prog array returns an id, - // not an fd. id := internal.NativeEndian.Uint32(buf) return NewMapFromID(MapID(id)) } -// MarshalBinary implements BinaryMarshaler. -func (m *Map) MarshalBinary() ([]byte, error) { +// marshalMap marshals the fd of a map into a buffer in host endianness. +func marshalMap(m *Map, length int) ([]byte, error) { + if length != 4 { + return nil, fmt.Errorf("can't marshal map to %d bytes", length) + } + fd, err := m.fd.Value() if err != nil { return nil, err @@ -690,8 +1081,8 @@ type MapIterator struct { func newMapIterator(target *Map) *MapIterator { return &MapIterator{ target: target, - maxEntries: target.abi.MaxEntries, - prevBytes: make([]byte, int(target.abi.KeySize)), + maxEntries: target.maxEntries, + prevBytes: make([]byte, target.keySize), } } @@ -710,7 +1101,9 @@ func (mi *MapIterator) Next(keyOut, valueOut interface{}) bool { return false } - for ; mi.count < mi.maxEntries; mi.count++ { + // For array-like maps NextKeyBytes returns nil only on after maxEntries + // iterations. + for mi.count <= mi.maxEntries { var nextBytes []byte nextBytes, mi.err = mi.target.NextKeyBytes(mi.prevKey) if mi.err != nil { @@ -729,6 +1122,7 @@ func (mi *MapIterator) Next(keyOut, valueOut interface{}) bool { copy(mi.prevBytes, nextBytes) mi.prevKey = mi.prevBytes + mi.count++ mi.err = mi.target.Lookup(nextBytes, valueOut) if errors.Is(mi.err, ErrKeyNotExist) { // Even though the key should be valid, we couldn't look up @@ -745,7 +1139,7 @@ func (mi *MapIterator) Next(keyOut, valueOut interface{}) bool { return false } - mi.err = unmarshalBytes(keyOut, nextBytes) + mi.err = mi.target.unmarshalKey(keyOut, nextBytes) return mi.err == nil } @@ -779,16 +1173,12 @@ func NewMapFromID(id MapID) (*Map, error) { return nil, err } - name, abi, err := newMapABIFromFd(fd) - if err != nil { - _ = fd.Close() - return nil, err - } - - return newMap(fd, name, abi) + return newMapFromFD(fd) } // ID returns the systemwide unique ID of the map. +// +// Deprecated: use MapInfo.ID() instead. func (m *Map) ID() (MapID, error) { info, err := bpfGetMapInfoByFD(m.fd) if err != nil { diff --git a/vendor/github.com/cilium/ebpf/marshalers.go b/vendor/github.com/cilium/ebpf/marshalers.go index c0db2f6b0b1c5..3ea1021a8d39e 100644 --- a/vendor/github.com/cilium/ebpf/marshalers.go +++ b/vendor/github.com/cilium/ebpf/marshalers.go @@ -13,14 +13,12 @@ import ( "github.com/cilium/ebpf/internal" ) +// marshalPtr converts an arbitrary value into a pointer suitable +// to be passed to the kernel. +// +// As an optimization, it returns the original value if it is an +// unsafe.Pointer. func marshalPtr(data interface{}, length int) (internal.Pointer, error) { - if data == nil { - if length == 0 { - return internal.NewPointer(nil), nil - } - return internal.Pointer{}, errors.New("can't use nil as key of map") - } - if ptr, ok := data.(unsafe.Pointer); ok { return internal.NewPointer(ptr), nil } @@ -33,6 +31,13 @@ func marshalPtr(data interface{}, length int) (internal.Pointer, error) { return internal.NewSlicePointer(buf), nil } +// marshalBytes converts an arbitrary value into a byte buffer. +// +// Prefer using Map.marshalKey and Map.marshalValue if possible, since +// those have special cases that allow more types to be encoded. +// +// Returns an error if the given value isn't representable in exactly +// length bytes. func marshalBytes(data interface{}, length int) (buf []byte, err error) { switch value := data.(type) { case encoding.BinaryMarshaler: @@ -43,6 +48,8 @@ func marshalBytes(data interface{}, length int) (buf []byte, err error) { buf = value case unsafe.Pointer: err = errors.New("can't marshal from unsafe.Pointer") + case Map, *Map, Program, *Program: + err = fmt.Errorf("can't marshal %T", value) default: var wr bytes.Buffer err = binary.Write(&wr, internal.NativeEndian, value) @@ -70,6 +77,10 @@ func makeBuffer(dst interface{}, length int) (internal.Pointer, []byte) { return internal.NewSlicePointer(buf), buf } +// unmarshalBytes converts a byte buffer into an arbitrary value. +// +// Prefer using Map.unmarshalKey and Map.unmarshalValue if possible, since +// those have special cases that allow more types to be encoded. func unmarshalBytes(data interface{}, buf []byte) error { switch value := data.(type) { case unsafe.Pointer: @@ -83,6 +94,8 @@ func unmarshalBytes(data interface{}, buf []byte) error { copy(dst, buf) runtime.KeepAlive(value) return nil + case Map, *Map, Program, *Program: + return fmt.Errorf("can't unmarshal into %T", value) case encoding.BinaryUnmarshaler: return value.UnmarshalBinary(buf) case *string: diff --git a/vendor/github.com/cilium/ebpf/pinning.go b/vendor/github.com/cilium/ebpf/pinning.go new file mode 100644 index 0000000000000..78812364ac700 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/pinning.go @@ -0,0 +1,42 @@ +package ebpf + +import ( + "errors" + "fmt" + "os" + + "github.com/cilium/ebpf/internal" +) + +func pin(currentPath, newPath string, fd *internal.FD) error { + if newPath == "" { + return errors.New("given pinning path cannot be empty") + } + if currentPath == "" { + return internal.BPFObjPin(newPath, fd) + } + if currentPath == newPath { + return nil + } + var err error + // Object is now moved to the new pinning path. + if err = os.Rename(currentPath, newPath); err == nil { + return nil + } + if !os.IsNotExist(err) { + return fmt.Errorf("unable to move pinned object to new path %v: %w", newPath, err) + } + // Internal state not in sync with the file system so let's fix it. + return internal.BPFObjPin(newPath, fd) +} + +func unpin(pinnedPath string) error { + if pinnedPath == "" { + return nil + } + err := os.Remove(pinnedPath) + if err == nil || os.IsNotExist(err) { + return nil + } + return err +} diff --git a/vendor/github.com/cilium/ebpf/prog.go b/vendor/github.com/cilium/ebpf/prog.go index 429203f07e850..4b65f23b20686 100644 --- a/vendor/github.com/cilium/ebpf/prog.go +++ b/vendor/github.com/cilium/ebpf/prog.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "math" + "path/filepath" "strings" "time" @@ -18,7 +19,7 @@ import ( // ErrNotSupported is returned whenever the kernel doesn't support a feature. var ErrNotSupported = internal.ErrNotSupported -// ProgramID represents the unique ID of an eBPF program +// ProgramID represents the unique ID of an eBPF program. type ProgramID uint32 const ( @@ -42,7 +43,7 @@ type ProgramOptions struct { LogSize int } -// ProgramSpec defines a Program +// ProgramSpec defines a Program. type ProgramSpec struct { // Name is passed to the kernel as a debug aid. Must only contain // alpha numeric and '_' characters. @@ -87,6 +88,13 @@ func (ps *ProgramSpec) Copy() *ProgramSpec { return &cpy } +// Tag calculates the kernel tag for a series of instructions. +// +// Use asm.Instructions.Tag if you need to calculate for non-native endianness. +func (ps *ProgramSpec) Tag() (string, error) { + return ps.Instructions.Tag(internal.NativeEndian) +} + // Program represents BPF program loaded into the kernel. // // It is not safe to close a Program which is used by other goroutines. @@ -97,8 +105,8 @@ type Program struct { fd *internal.FD name string - abi ProgramABI - attachType AttachType + pinnedPath string + typ ProgramType } // NewProgram creates a new Program. @@ -114,24 +122,98 @@ func NewProgram(spec *ProgramSpec) (*Program, error) { // Loading a program for the first time will perform // feature detection by loading small, temporary programs. func NewProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, error) { - if spec.BTF == nil { - return newProgramWithBTF(spec, nil, opts) + btfs := make(btfHandleCache) + defer btfs.close() + + return newProgramWithOptions(spec, opts, btfs) +} + +func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, btfs btfHandleCache) (*Program, error) { + if len(spec.Instructions) == 0 { + return nil, errors.New("Instructions cannot be empty") } - handle, err := btf.NewHandle(btf.ProgramSpec(spec.BTF)) - if err != nil && !errors.Is(err, btf.ErrNotSupported) { - return nil, fmt.Errorf("can't load BTF: %w", err) + if len(spec.License) == 0 { + return nil, errors.New("License cannot be empty") } - return newProgramWithBTF(spec, handle, opts) -} + if spec.ByteOrder != nil && spec.ByteOrder != internal.NativeEndian { + return nil, fmt.Errorf("can't load %s program on %s", spec.ByteOrder, internal.NativeEndian) + } -func newProgramWithBTF(spec *ProgramSpec, btf *btf.Handle, opts ProgramOptions) (*Program, error) { - attr, err := convertProgramSpec(spec, btf) + insns := make(asm.Instructions, len(spec.Instructions)) + copy(insns, spec.Instructions) + + if err := fixupJumpsAndCalls(insns); err != nil { + return nil, err + } + + buf := bytes.NewBuffer(make([]byte, 0, len(spec.Instructions)*asm.InstructionSize)) + err := insns.Marshal(buf, internal.NativeEndian) if err != nil { return nil, err } + bytecode := buf.Bytes() + insCount := uint32(len(bytecode) / asm.InstructionSize) + attr := &bpfProgLoadAttr{ + progType: spec.Type, + expectedAttachType: spec.AttachType, + insCount: insCount, + instructions: internal.NewSlicePointer(bytecode), + license: internal.NewStringPointer(spec.License), + kernelVersion: spec.KernelVersion, + } + + if haveObjName() == nil { + attr.progName = newBPFObjName(spec.Name) + } + + var btfDisabled bool + if spec.BTF != nil { + if relos, err := btf.ProgramRelocations(spec.BTF, nil); err != nil { + return nil, fmt.Errorf("CO-RE relocations: %s", err) + } else if len(relos) > 0 { + return nil, fmt.Errorf("applying CO-RE relocations: %w", ErrNotSupported) + } + + handle, err := btfs.load(btf.ProgramSpec(spec.BTF)) + btfDisabled = errors.Is(err, btf.ErrNotSupported) + if err != nil && !btfDisabled { + return nil, fmt.Errorf("load BTF: %w", err) + } + + if handle != nil { + attr.progBTFFd = uint32(handle.FD()) + + recSize, bytes, err := btf.ProgramLineInfos(spec.BTF) + if err != nil { + return nil, fmt.Errorf("get BTF line infos: %w", err) + } + attr.lineInfoRecSize = recSize + attr.lineInfoCnt = uint32(uint64(len(bytes)) / uint64(recSize)) + attr.lineInfo = internal.NewSlicePointer(bytes) + + recSize, bytes, err = btf.ProgramFuncInfos(spec.BTF) + if err != nil { + return nil, fmt.Errorf("get BTF function infos: %w", err) + } + attr.funcInfoRecSize = recSize + attr.funcInfoCnt = uint32(uint64(len(bytes)) / uint64(recSize)) + attr.funcInfo = internal.NewSlicePointer(bytes) + } + } + + if spec.AttachTo != "" { + target, err := resolveBTFType(spec.AttachTo, spec.Type, spec.AttachType) + if err != nil { + return nil, err + } + if target != nil { + attr.attachBTFID = target.ID() + } + } + logSize := DefaultVerifierLogSize if opts.LogSize > 0 { logSize = opts.LogSize @@ -147,9 +229,7 @@ func newProgramWithBTF(spec *ProgramSpec, btf *btf.Handle, opts ProgramOptions) fd, err := bpfProgLoad(attr) if err == nil { - prog := newProgram(fd, spec.Name, &ProgramABI{spec.Type}) - prog.VerifierLog = internal.CString(logBuf) - return prog, nil + return &Program{internal.CString(logBuf), fd, spec.Name, "", spec.Type}, nil } logErr := err @@ -163,115 +243,71 @@ func newProgramWithBTF(spec *ProgramSpec, btf *btf.Handle, opts ProgramOptions) _, logErr = bpfProgLoad(attr) } + if errors.Is(logErr, unix.EPERM) && logBuf[0] == 0 { + // EPERM due to RLIMIT_MEMLOCK happens before the verifier, so we can + // check that the log is empty to reduce false positives. + return nil, fmt.Errorf("load program: RLIMIT_MEMLOCK may be too low: %w", logErr) + } + err = internal.ErrorWithLog(err, logBuf, logErr) - return nil, fmt.Errorf("can't load program: %w", err) + if btfDisabled { + return nil, fmt.Errorf("load program without BTF: %w", err) + } + return nil, fmt.Errorf("load program: %w", err) } // NewProgramFromFD creates a program from a raw fd. // // You should not use fd after calling this function. // -// Requires at least Linux 4.11. +// Requires at least Linux 4.10. func NewProgramFromFD(fd int) (*Program, error) { if fd < 0 { return nil, errors.New("invalid fd") } - bpfFd := internal.NewFD(uint32(fd)) - name, abi, err := newProgramABIFromFd(bpfFd) - if err != nil { - bpfFd.Forget() - return nil, err - } - - return newProgram(bpfFd, name, abi), nil + return newProgramFromFD(internal.NewFD(uint32(fd))) } -func newProgram(fd *internal.FD, name string, abi *ProgramABI) *Program { - return &Program{ - name: name, - fd: fd, - abi: *abi, - } -} - -func convertProgramSpec(spec *ProgramSpec, handle *btf.Handle) (*bpfProgLoadAttr, error) { - if len(spec.Instructions) == 0 { - return nil, errors.New("Instructions cannot be empty") - } - - if len(spec.License) == 0 { - return nil, errors.New("License cannot be empty") - } - - if spec.ByteOrder != nil && spec.ByteOrder != internal.NativeEndian { - return nil, fmt.Errorf("can't load %s program on %s", spec.ByteOrder, internal.NativeEndian) - } - - buf := bytes.NewBuffer(make([]byte, 0, len(spec.Instructions)*asm.InstructionSize)) - err := spec.Instructions.Marshal(buf, internal.NativeEndian) +// NewProgramFromID returns the program for a given id. +// +// Returns ErrNotExist, if there is no eBPF program with the given id. +func NewProgramFromID(id ProgramID) (*Program, error) { + fd, err := bpfObjGetFDByID(internal.BPF_PROG_GET_FD_BY_ID, uint32(id)) if err != nil { - return nil, err + return nil, fmt.Errorf("get program by id: %w", err) } - bytecode := buf.Bytes() - insCount := uint32(len(bytecode) / asm.InstructionSize) - attr := &bpfProgLoadAttr{ - progType: spec.Type, - expectedAttachType: spec.AttachType, - insCount: insCount, - instructions: internal.NewSlicePointer(bytecode), - license: internal.NewStringPointer(spec.License), - kernelVersion: spec.KernelVersion, - } - - if haveObjName() == nil { - attr.progName = newBPFObjName(spec.Name) - } - - if handle != nil && spec.BTF != nil { - attr.progBTFFd = uint32(handle.FD()) - - recSize, bytes, err := btf.ProgramLineInfos(spec.BTF) - if err != nil { - return nil, fmt.Errorf("can't get BTF line infos: %w", err) - } - attr.lineInfoRecSize = recSize - attr.lineInfoCnt = uint32(uint64(len(bytes)) / uint64(recSize)) - attr.lineInfo = internal.NewSlicePointer(bytes) - - recSize, bytes, err = btf.ProgramFuncInfos(spec.BTF) - if err != nil { - return nil, fmt.Errorf("can't get BTF function infos: %w", err) - } - attr.funcInfoRecSize = recSize - attr.funcInfoCnt = uint32(uint64(len(bytes)) / uint64(recSize)) - attr.funcInfo = internal.NewSlicePointer(bytes) - } + return newProgramFromFD(fd) +} - if spec.AttachTo != "" { - target, err := resolveBTFType(spec.AttachTo, spec.Type, spec.AttachType) - if err != nil { - return nil, err - } - if target != nil { - attr.attachBTFID = target.ID() - } +func newProgramFromFD(fd *internal.FD) (*Program, error) { + info, err := newProgramInfoFromFd(fd) + if err != nil { + fd.Close() + return nil, fmt.Errorf("discover program type: %w", err) } - return attr, nil + return &Program{"", fd, "", "", info.Type}, nil } func (p *Program) String() string { if p.name != "" { - return fmt.Sprintf("%s(%s)#%v", p.abi.Type, p.name, p.fd) + return fmt.Sprintf("%s(%s)#%v", p.typ, p.name, p.fd) } - return fmt.Sprintf("%s#%v", p.abi.Type, p.fd) + return fmt.Sprintf("%s(%v)", p.typ, p.fd) } -// ABI gets the ABI of the Program -func (p *Program) ABI() ProgramABI { - return p.abi +// Type returns the underlying type of the program. +func (p *Program) Type() ProgramType { + return p.typ +} + +// Info returns metadata about the program. +// +// Requires at least 4.10. +func (p *Program) Info() (*ProgramInfo, error) { + return newProgramInfoFromFd(p.fd) } // FD gets the file descriptor of the Program. @@ -303,19 +339,42 @@ func (p *Program) Clone() (*Program, error) { return nil, fmt.Errorf("can't clone program: %w", err) } - return newProgram(dup, p.name, &p.abi), nil + return &Program{p.VerifierLog, dup, p.name, "", p.typ}, nil } -// Pin persists the Program past the lifetime of the process that created it +// Pin persists the Program on the BPF virtual file system past the lifetime of +// the process that created it // -// This requires bpffs to be mounted above fileName. See http://cilium.readthedocs.io/en/doc-1.0/kubernetes/install/#mounting-the-bpf-fs-optional +// This requires bpffs to be mounted above fileName. See https://docs.cilium.io/en/k8s-doc/admin/#admin-mount-bpffs func (p *Program) Pin(fileName string) error { - if err := internal.BPFObjPin(fileName, p.fd); err != nil { - return fmt.Errorf("can't pin program: %w", err) + if err := pin(p.pinnedPath, fileName, p.fd); err != nil { + return err + } + p.pinnedPath = fileName + return nil +} + +// Unpin removes the persisted state for the Program from the BPF virtual filesystem. +// +// Failed calls to Unpin will not alter the state returned by IsPinned. +// +// Unpinning an unpinned Program returns nil. +func (p *Program) Unpin() error { + if err := unpin(p.pinnedPath); err != nil { + return err } + p.pinnedPath = "" return nil } +// IsPinned returns true if the Program has a non-empty pinned path. +func (p *Program) IsPinned() bool { + if p.pinnedPath == "" { + return false + } + return true +} + // Close unloads the program from the kernel. func (p *Program) Close() error { if p == nil { @@ -359,7 +418,7 @@ func (p *Program) Benchmark(in []byte, repeat int, reset func()) (uint32, time.D return ret, total, nil } -var haveProgTestRun = internal.FeatureTest("BPF_PROG_TEST_RUN", "4.12", func() (bool, error) { +var haveProgTestRun = internal.FeatureTest("BPF_PROG_TEST_RUN", "4.12", func() error { prog, err := NewProgram(&ProgramSpec{ Type: SocketFilter, Instructions: asm.Instructions{ @@ -370,7 +429,7 @@ var haveProgTestRun = internal.FeatureTest("BPF_PROG_TEST_RUN", "4.12", func() ( }) if err != nil { // This may be because we lack sufficient permissions, etc. - return false, err + return err } defer prog.Close() @@ -383,10 +442,16 @@ var haveProgTestRun = internal.FeatureTest("BPF_PROG_TEST_RUN", "4.12", func() ( } err = bpfProgTestRun(&attr) - - // Check for EINVAL specifically, rather than err != nil since we - // otherwise misdetect due to insufficient permissions. - return !errors.Is(err, unix.EINVAL), nil + if errors.Is(err, unix.EINVAL) { + // Check for EINVAL specifically, rather than err != nil since we + // otherwise misdetect due to insufficient permissions. + return internal.ErrNotSupported + } + if errors.Is(err, unix.EINTR) { + // We know that PROG_TEST_RUN is supported if we get EINTR. + return nil + } + return err }) func (p *Program) testRun(in []byte, repeat int, reset func()) (uint32, []byte, time.Duration, error) { @@ -465,8 +530,11 @@ func unmarshalProgram(buf []byte) (*Program, error) { return NewProgramFromID(ProgramID(id)) } -// MarshalBinary implements BinaryMarshaler. -func (p *Program) MarshalBinary() ([]byte, error) { +func marshalProgram(p *Program, length int) ([]byte, error) { + if length != 4 { + return nil, fmt.Errorf("can't marshal program to %d bytes", length) + } + value, err := p.fd.Value() if err != nil { return nil, err @@ -535,22 +603,22 @@ func LoadPinnedProgram(fileName string) (*Program, error) { return nil, err } - name, abi, err := newProgramABIFromFd(fd) + info, err := newProgramInfoFromFd(fd) if err != nil { _ = fd.Close() - return nil, fmt.Errorf("can't get ABI for %s: %w", fileName, err) + return nil, fmt.Errorf("info for %s: %w", fileName, err) } - return newProgram(fd, name, abi), nil + return &Program{"", fd, filepath.Base(fileName), "", info.Type}, nil } -// SanitizeName replaces all invalid characters in name. +// SanitizeName replaces all invalid characters in name with replacement. +// Passing a negative value for replacement will delete characters instead +// of replacing them. Use this to automatically generate valid names for maps +// and programs at runtime. // -// Use this to automatically generate valid names for maps and -// programs at run time. -// -// Passing a negative value for replacement will delete characters -// instead of replacing them. +// The set of allowed characters depends on the running kernel version. +// Dots are only allowed as of kernel 5.2. func SanitizeName(name string, replacement rune) string { return strings.Map(func(char rune) rune { if invalidBPFObjNameChar(char) { @@ -568,25 +636,9 @@ func ProgramGetNextID(startID ProgramID) (ProgramID, error) { return ProgramID(id), err } -// NewProgramFromID returns the program for a given id. -// -// Returns ErrNotExist, if there is no eBPF program with the given id. -func NewProgramFromID(id ProgramID) (*Program, error) { - fd, err := bpfObjGetFDByID(internal.BPF_PROG_GET_FD_BY_ID, uint32(id)) - if err != nil { - return nil, err - } - - name, abi, err := newProgramABIFromFd(fd) - if err != nil { - _ = fd.Close() - return nil, err - } - - return newProgram(fd, name, abi), nil -} - // ID returns the systemwide unique ID of the program. +// +// Deprecated: use ProgramInfo.ID() instead. func (p *Program) ID() (ProgramID, error) { info, err := bpfGetProgInfoByFD(p.fd) if err != nil { @@ -595,12 +647,16 @@ func (p *Program) ID() (ProgramID, error) { return ProgramID(info.id), nil } -func resolveBTFType(name string, progType ProgramType, attachType AttachType) (btf.Type, error) { +func findKernelType(name string, typ btf.Type) error { kernel, err := btf.LoadKernelSpec() if err != nil { - return nil, fmt.Errorf("can't resolve BTF type %s: %w", name, err) + return fmt.Errorf("can't load kernel spec: %w", err) } + return kernel.FindType(name, typ) +} + +func resolveBTFType(name string, progType ProgramType, attachType AttachType) (btf.Type, error) { type match struct { p ProgramType a AttachType @@ -608,10 +664,30 @@ func resolveBTFType(name string, progType ProgramType, attachType AttachType) (b target := match{progType, attachType} switch target { + case match{LSM, AttachLSMMac}: + var target btf.Func + err := findKernelType("bpf_lsm_"+name, &target) + if errors.Is(err, btf.ErrNotFound) { + return nil, &internal.UnsupportedFeatureError{ + Name: name + " LSM hook", + } + } + if err != nil { + return nil, fmt.Errorf("resolve BTF for LSM hook %s: %w", name, err) + } + + return &target, nil + case match{Tracing, AttachTraceIter}: var target btf.Func - if err := kernel.FindType("bpf_iter_"+name, &target); err != nil { - return nil, fmt.Errorf("can't resolve BTF for iterator %s: %w", name, err) + err := findKernelType("bpf_iter_"+name, &target) + if errors.Is(err, btf.ErrNotFound) { + return nil, &internal.UnsupportedFeatureError{ + Name: name + " iterator", + } + } + if err != nil { + return nil, fmt.Errorf("resolve BTF for iterator %s: %w", name, err) } return &target, nil diff --git a/vendor/github.com/cilium/ebpf/readme.md b/vendor/github.com/cilium/ebpf/readme.md deleted file mode 100644 index cc8e89c75a957..0000000000000 --- a/vendor/github.com/cilium/ebpf/readme.md +++ /dev/null @@ -1,25 +0,0 @@ -eBPF -------- -[![](https://godoc.org/github.com/cilium/ebpf?status.svg)](https://godoc.org/github.com/cilium/ebpf) - -eBPF is a pure Go library that provides utilities for loading, compiling, and debugging eBPF programs. It has minimal external dependencies and is intended to be used in long running processes. - -[ebpf/asm](https://godoc.org/github.com/cilium/ebpf/asm) contains a basic assembler. - -The library is maintained by [Cloudflare](https://www.cloudflare.com) and [Cilium](https://www.cilium.io). Feel free to [join](https://cilium.herokuapp.com/) the [libbpf-go](https://cilium.slack.com/messages/libbpf-go) channel on Slack. - -## Current status - -The package is production ready, but **the API is explicitly unstable -right now**. Expect to update your code if you want to follow along. - -## Requirements - -* A version of Go that is [supported by upstream](https://golang.org/doc/devel/release.html#policy) -* Linux 4.9, 4.19 or 5.4 (versions in-between should work, but are not tested) - -## Useful resources - -* [Cilium eBPF documentation](https://cilium.readthedocs.io/en/latest/bpf/#bpf-guide) (recommended) -* [Linux documentation on BPF](http://elixir.free-electrons.com/linux/latest/source/Documentation/networking/filter.txt) -* [eBPF features by Linux version](https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md) diff --git a/vendor/github.com/cilium/ebpf/syscalls.go b/vendor/github.com/cilium/ebpf/syscalls.go index 2b713d06a0c75..1cba1d747a4ea 100644 --- a/vendor/github.com/cilium/ebpf/syscalls.go +++ b/vendor/github.com/cilium/ebpf/syscalls.go @@ -3,7 +3,6 @@ package ebpf import ( "errors" "fmt" - "os" "unsafe" "github.com/cilium/ebpf/internal" @@ -12,9 +11,7 @@ import ( ) // Generic errors returned by BPF syscalls. -var ( - ErrNotExist = errors.New("requested object does not exist") -) +var ErrNotExist = errors.New("requested object does not exist") // bpfObjName is a null-terminated string made up of // 'A-Za-z0-9_' characters. @@ -27,18 +24,20 @@ func newBPFObjName(name string) bpfObjName { return result } +// invalidBPFObjNameChar returns true if char may not appear in +// a BPF object name. func invalidBPFObjNameChar(char rune) bool { dotAllowed := objNameAllowsDot() == nil switch { case char >= 'A' && char <= 'Z': - fallthrough + return false case char >= 'a' && char <= 'z': - fallthrough + return false case char >= '0' && char <= '9': - fallthrough + return false case dotAllowed && char == '.': - fallthrough + return false case char == '_': return false default: @@ -69,14 +68,32 @@ type bpfMapOpAttr struct { flags uint64 } +type bpfBatchMapOpAttr struct { + inBatch internal.Pointer + outBatch internal.Pointer + keys internal.Pointer + values internal.Pointer + count uint32 + mapFd uint32 + elemFlags uint64 + flags uint64 +} + type bpfMapInfo struct { - mapType uint32 - id uint32 - keySize uint32 - valueSize uint32 - maxEntries uint32 - flags uint32 - mapName bpfObjName // since 4.15 ad5b177bd73f + map_type uint32 // since 4.12 1e2709769086 + id uint32 + key_size uint32 + value_size uint32 + max_entries uint32 + map_flags uint32 + name bpfObjName // since 4.15 ad5b177bd73f + ifindex uint32 // since 4.16 52775b33bb50 + btf_vmlinux_value_type_id uint32 // since 5.6 85d33df357b6 + netns_dev uint64 // since 4.16 52775b33bb50 + netns_ino uint64 + btf_id uint32 // since 4.18 78958fca7ead + btf_key_type_id uint32 // since 4.18 9b2cf328b2ec + btf_value_type_id uint32 } type bpfProgLoadAttr struct { @@ -104,18 +121,40 @@ type bpfProgLoadAttr struct { } type bpfProgInfo struct { - progType uint32 - id uint32 - tag [unix.BPF_TAG_SIZE]byte - jitedLen uint32 - xlatedLen uint32 - jited internal.Pointer - xlated internal.Pointer - loadTime uint64 // since 4.15 cb4d2b3f03d8 - createdByUID uint32 - nrMapIDs uint32 - mapIds internal.Pointer - name bpfObjName + prog_type uint32 + id uint32 + tag [unix.BPF_TAG_SIZE]byte + jited_prog_len uint32 + xlated_prog_len uint32 + jited_prog_insns internal.Pointer + xlated_prog_insns internal.Pointer + load_time uint64 // since 4.15 cb4d2b3f03d8 + created_by_uid uint32 + nr_map_ids uint32 + map_ids internal.Pointer + name bpfObjName // since 4.15 067cae47771c + ifindex uint32 + gpl_compatible uint32 + netns_dev uint64 + netns_ino uint64 + nr_jited_ksyms uint32 + nr_jited_func_lens uint32 + jited_ksyms internal.Pointer + jited_func_lens internal.Pointer + btf_id uint32 + func_info_rec_size uint32 + func_info internal.Pointer + nr_func_info uint32 + nr_line_info uint32 + line_info internal.Pointer + jited_line_info internal.Pointer + nr_jited_line_info uint32 + line_info_rec_size uint32 + jited_line_info_rec_size uint32 + nr_prog_tags uint32 + prog_tags internal.Pointer + run_time_ns uint64 + run_cnt uint64 } type bpfProgTestRunAttr struct { @@ -129,12 +168,6 @@ type bpfProgTestRunAttr struct { duration uint32 } -type bpfObjGetInfoByFDAttr struct { - fd uint32 - infoLen uint32 - info internal.Pointer // May be either bpfMapInfo or bpfProgInfo -} - type bpfGetFDByIDAttr struct { id uint32 next uint32 @@ -174,10 +207,6 @@ func bpfProgTestRun(attr *bpfProgTestRunAttr) error { func bpfMapCreate(attr *bpfMapCreateAttr) (*internal.FD, error) { fd, err := internal.BPF(internal.BPF_MAP_CREATE, unsafe.Pointer(attr), unsafe.Sizeof(*attr)) - if errors.Is(err, os.ErrPermission) { - return nil, errors.New("permission denied or insufficient rlimit to lock memory for map") - } - if err != nil { return nil, err } @@ -185,35 +214,25 @@ func bpfMapCreate(attr *bpfMapCreateAttr) (*internal.FD, error) { return internal.NewFD(uint32(fd)), nil } -var haveNestedMaps = internal.FeatureTest("nested maps", "4.12", func() (bool, error) { - inner, err := bpfMapCreate(&bpfMapCreateAttr{ - mapType: Array, - keySize: 4, - valueSize: 4, - maxEntries: 1, - }) - if err != nil { - return false, err - } - defer inner.Close() - - innerFd, _ := inner.Value() - nested, err := bpfMapCreate(&bpfMapCreateAttr{ +var haveNestedMaps = internal.FeatureTest("nested maps", "4.12", func() error { + _, err := bpfMapCreate(&bpfMapCreateAttr{ mapType: ArrayOfMaps, keySize: 4, valueSize: 4, maxEntries: 1, - innerMapFd: innerFd, + // Invalid file descriptor. + innerMapFd: ^uint32(0), }) - if err != nil { - return false, nil + if errors.Is(err, unix.EINVAL) { + return internal.ErrNotSupported } - - _ = nested.Close() - return true, nil + if errors.Is(err, unix.EBADF) { + return nil + } + return err }) -var haveMapMutabilityModifiers = internal.FeatureTest("read- and write-only maps", "5.2", func() (bool, error) { +var haveMapMutabilityModifiers = internal.FeatureTest("read- and write-only maps", "5.2", func() error { // This checks BPF_F_RDONLY_PROG and BPF_F_WRONLY_PROG. Since // BPF_MAP_FREEZE appeared in 5.2 as well we don't do a separate check. m, err := bpfMapCreate(&bpfMapCreateAttr{ @@ -224,10 +243,10 @@ var haveMapMutabilityModifiers = internal.FeatureTest("read- and write-only maps flags: unix.BPF_F_RDONLY_PROG, }) if err != nil { - return false, nil + return internal.ErrNotSupported } _ = m.Close() - return true, nil + return nil }) func bpfMapLookupElem(m *internal.FD, key, valueOut internal.Pointer) error { @@ -313,6 +332,29 @@ func objGetNextID(cmd internal.BPFCmd, start uint32) (uint32, error) { return attr.nextID, wrapObjError(err) } +func bpfMapBatch(cmd internal.BPFCmd, m *internal.FD, inBatch, outBatch, keys, values internal.Pointer, count uint32, opts *BatchOptions) (uint32, error) { + fd, err := m.Value() + if err != nil { + return 0, err + } + + attr := bpfBatchMapOpAttr{ + inBatch: inBatch, + outBatch: outBatch, + keys: keys, + values: values, + count: count, + mapFd: fd, + } + if opts != nil { + attr.elemFlags = opts.ElemFlags + attr.flags = opts.Flags + } + _, err = internal.BPF(cmd, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) + // always return count even on an error, as things like update might partially be fulfilled. + return attr.count, wrapMapError(err) +} + func wrapObjError(err error) error { if err == nil { return nil @@ -337,6 +379,10 @@ func wrapMapError(err error) error { return ErrKeyExist } + if errors.Is(err, unix.ENOTSUPP) { + return ErrNotSupported + } + return errors.New(err.Error()) } @@ -353,28 +399,9 @@ func bpfMapFreeze(m *internal.FD) error { return err } -func bpfGetObjectInfoByFD(fd *internal.FD, info unsafe.Pointer, size uintptr) error { - value, err := fd.Value() - if err != nil { - return err - } - - // available from 4.13 - attr := bpfObjGetInfoByFDAttr{ - fd: value, - infoLen: uint32(size), - info: internal.NewPointer(info), - } - _, err = internal.BPF(internal.BPF_OBJ_GET_INFO_BY_FD, unsafe.Pointer(&attr), unsafe.Sizeof(attr)) - if err != nil { - return fmt.Errorf("fd %d: %w", fd, err) - } - return nil -} - func bpfGetProgInfoByFD(fd *internal.FD) (*bpfProgInfo, error) { var info bpfProgInfo - if err := bpfGetObjectInfoByFD(fd, unsafe.Pointer(&info), unsafe.Sizeof(info)); err != nil { + if err := internal.BPFObjGetInfoByFD(fd, unsafe.Pointer(&info), unsafe.Sizeof(info)); err != nil { return nil, fmt.Errorf("can't get program info: %w", err) } return &info, nil @@ -382,14 +409,14 @@ func bpfGetProgInfoByFD(fd *internal.FD) (*bpfProgInfo, error) { func bpfGetMapInfoByFD(fd *internal.FD) (*bpfMapInfo, error) { var info bpfMapInfo - err := bpfGetObjectInfoByFD(fd, unsafe.Pointer(&info), unsafe.Sizeof(info)) + err := internal.BPFObjGetInfoByFD(fd, unsafe.Pointer(&info), unsafe.Sizeof(info)) if err != nil { return nil, fmt.Errorf("can't get map info: %w", err) } return &info, nil } -var haveObjName = internal.FeatureTest("object names", "4.15", func() (bool, error) { +var haveObjName = internal.FeatureTest("object names", "4.15", func() error { attr := bpfMapCreateAttr{ mapType: Array, keySize: 4, @@ -400,16 +427,16 @@ var haveObjName = internal.FeatureTest("object names", "4.15", func() (bool, err fd, err := bpfMapCreate(&attr) if err != nil { - return false, nil + return internal.ErrNotSupported } _ = fd.Close() - return true, nil + return nil }) -var objNameAllowsDot = internal.FeatureTest("dot in object names", "5.2", func() (bool, error) { +var objNameAllowsDot = internal.FeatureTest("dot in object names", "5.2", func() error { if err := haveObjName(); err != nil { - return false, err + return err } attr := bpfMapCreateAttr{ @@ -422,11 +449,37 @@ var objNameAllowsDot = internal.FeatureTest("dot in object names", "5.2", func() fd, err := bpfMapCreate(&attr) if err != nil { - return false, nil + return internal.ErrNotSupported } _ = fd.Close() - return true, nil + return nil +}) + +var haveBatchAPI = internal.FeatureTest("map batch api", "5.6", func() error { + var maxEntries uint32 = 2 + attr := bpfMapCreateAttr{ + mapType: Hash, + keySize: 4, + valueSize: 4, + maxEntries: maxEntries, + } + + fd, err := bpfMapCreate(&attr) + if err != nil { + return internal.ErrNotSupported + } + defer fd.Close() + keys := []uint32{1, 2} + values := []uint32{3, 4} + kp, _ := marshalPtr(keys, 8) + vp, _ := marshalPtr(values, 8) + nilPtr := internal.NewPointer(nil) + _, err = bpfMapBatch(internal.BPF_MAP_UPDATE_BATCH, fd, nilPtr, nilPtr, kp, vp, maxEntries, nil) + if err != nil { + return internal.ErrNotSupported + } + return nil }) func bpfObjGetFDByID(cmd internal.BPFCmd, id uint32) (*internal.FD, error) { diff --git a/vendor/github.com/cilium/ebpf/types.go b/vendor/github.com/cilium/ebpf/types.go index 1ffc62123be25..3191ba1e027d2 100644 --- a/vendor/github.com/cilium/ebpf/types.go +++ b/vendor/github.com/cilium/ebpf/types.go @@ -1,6 +1,6 @@ package ebpf -//go:generate stringer -output types_string.go -type=MapType,ProgramType,AttachType +//go:generate stringer -output types_string.go -type=MapType,ProgramType,AttachType,PinType // MapType indicates the type map structure // that will be initialized in the kernel. @@ -85,10 +85,19 @@ const ( // hasPerCPUValue returns true if the Map stores a value per CPU. func (mt MapType) hasPerCPUValue() bool { - if mt == PerCPUHash || mt == PerCPUArray || mt == LRUCPUHash { - return true - } - return false + return mt == PerCPUHash || mt == PerCPUArray || mt == LRUCPUHash +} + +// canStoreMap returns true if the map type accepts a map fd +// for update and returns a map id for lookup. +func (mt MapType) canStoreMap() bool { + return mt == ArrayOfMaps || mt == HashOfMaps +} + +// canStoreProgram returns true if the map type accepts a program fd +// for update and returns a program id for lookup. +func (mt MapType) canStoreProgram() bool { + return mt == ProgramArray } // ProgramType of the eBPF program @@ -96,60 +105,37 @@ type ProgramType uint32 // eBPF program types const ( - // Unrecognized program type UnspecifiedProgram ProgramType = iota - // SocketFilter socket or seccomp filter SocketFilter - // Kprobe program Kprobe - // SchedCLS traffic control shaper SchedCLS - // SchedACT routing control shaper SchedACT - // TracePoint program TracePoint - // XDP program XDP - // PerfEvent program PerfEvent - // CGroupSKB program CGroupSKB - // CGroupSock program CGroupSock - // LWTIn program LWTIn - // LWTOut program LWTOut - // LWTXmit program LWTXmit - // SockOps program SockOps - // SkSKB program SkSKB - // CGroupDevice program CGroupDevice - // SkMsg program SkMsg - // RawTracepoint program RawTracepoint - // CGroupSockAddr program CGroupSockAddr - // LWTSeg6Local program LWTSeg6Local - // LircMode2 program LircMode2 - // SkReuseport program SkReuseport - // FlowDissector program FlowDissector - // CGroupSysctl program CGroupSysctl - // RawTracepointWritable program RawTracepointWritable - // CGroupSockopt program CGroupSockopt - // Tracing program Tracing + StructOps + Extension + LSM + SkLookup ) // AttachType of the eBPF program, needed to differentiate allowed context accesses in @@ -157,7 +143,7 @@ const ( // Will cause invalid argument (EINVAL) at program load time if set incorrectly. type AttachType uint32 -// AttachNone is an alias for AttachCGroupInetIngress for readability reasons +// AttachNone is an alias for AttachCGroupInetIngress for readability reasons. const AttachNone AttachType = 0 const ( @@ -190,7 +176,38 @@ const ( AttachModifyReturn AttachLSMMac AttachTraceIter + AttachCgroupInet4GetPeername + AttachCgroupInet6GetPeername + AttachCgroupInet4GetSockname + AttachCgroupInet6GetSockname + AttachXDPDevMap + AttachCgroupInetSockRelease + AttachXDPCPUMap + AttachSkLookup + AttachXDP ) // AttachFlags of the eBPF program used in BPF_PROG_ATTACH command type AttachFlags uint32 + +// PinType determines whether a map is pinned into a BPFFS. +type PinType int + +// Valid pin types. +// +// Mirrors enum libbpf_pin_type. +const ( + PinNone PinType = iota + // Pin an object by using its name as the filename. + PinByName +) + +// BatchOptions batch map operations options +// +// Mirrors libbpf struct bpf_map_batch_opts +// Currently BPF_F_FLAG is the only supported +// flag (for ElemFlags). +type BatchOptions struct { + ElemFlags uint64 + Flags uint64 +} diff --git a/vendor/github.com/cilium/ebpf/types_string.go b/vendor/github.com/cilium/ebpf/types_string.go index c7139578ec6f3..976bd76be0104 100644 --- a/vendor/github.com/cilium/ebpf/types_string.go +++ b/vendor/github.com/cilium/ebpf/types_string.go @@ -1,4 +1,4 @@ -// Code generated by "stringer -output types_string.go -type=MapType,ProgramType,AttachType"; DO NOT EDIT. +// Code generated by "stringer -output types_string.go -type=MapType,ProgramType,AttachType,PinType"; DO NOT EDIT. package ebpf @@ -77,11 +77,15 @@ func _() { _ = x[RawTracepointWritable-24] _ = x[CGroupSockopt-25] _ = x[Tracing-26] + _ = x[StructOps-27] + _ = x[Extension-28] + _ = x[LSM-29] + _ = x[SkLookup-30] } -const _ProgramType_name = "UnspecifiedProgramSocketFilterKprobeSchedCLSSchedACTTracePointXDPPerfEventCGroupSKBCGroupSockLWTInLWTOutLWTXmitSockOpsSkSKBCGroupDeviceSkMsgRawTracepointCGroupSockAddrLWTSeg6LocalLircMode2SkReuseportFlowDissectorCGroupSysctlRawTracepointWritableCGroupSockoptTracing" +const _ProgramType_name = "UnspecifiedProgramSocketFilterKprobeSchedCLSSchedACTTracePointXDPPerfEventCGroupSKBCGroupSockLWTInLWTOutLWTXmitSockOpsSkSKBCGroupDeviceSkMsgRawTracepointCGroupSockAddrLWTSeg6LocalLircMode2SkReuseportFlowDissectorCGroupSysctlRawTracepointWritableCGroupSockoptTracingStructOpsExtensionLSMSkLookup" -var _ProgramType_index = [...]uint16{0, 18, 30, 36, 44, 52, 62, 65, 74, 83, 93, 98, 104, 111, 118, 123, 135, 140, 153, 167, 179, 188, 199, 212, 224, 245, 258, 265} +var _ProgramType_index = [...]uint16{0, 18, 30, 36, 44, 52, 62, 65, 74, 83, 93, 98, 104, 111, 118, 123, 135, 140, 153, 167, 179, 188, 199, 212, 224, 245, 258, 265, 274, 283, 286, 294} func (i ProgramType) String() string { if i >= ProgramType(len(_ProgramType_index)-1) { @@ -123,11 +127,20 @@ func _() { _ = x[AttachModifyReturn-26] _ = x[AttachLSMMac-27] _ = x[AttachTraceIter-28] + _ = x[AttachCgroupInet4GetPeername-29] + _ = x[AttachCgroupInet6GetPeername-30] + _ = x[AttachCgroupInet4GetSockname-31] + _ = x[AttachCgroupInet6GetSockname-32] + _ = x[AttachXDPDevMap-33] + _ = x[AttachCgroupInetSockRelease-34] + _ = x[AttachXDPCPUMap-35] + _ = x[AttachSkLookup-36] + _ = x[AttachXDP-37] } -const _AttachType_name = "AttachNoneAttachCGroupInetEgressAttachCGroupInetSockCreateAttachCGroupSockOpsAttachSkSKBStreamParserAttachSkSKBStreamVerdictAttachCGroupDeviceAttachSkMsgVerdictAttachCGroupInet4BindAttachCGroupInet6BindAttachCGroupInet4ConnectAttachCGroupInet6ConnectAttachCGroupInet4PostBindAttachCGroupInet6PostBindAttachCGroupUDP4SendmsgAttachCGroupUDP6SendmsgAttachLircMode2AttachFlowDissectorAttachCGroupSysctlAttachCGroupUDP4RecvmsgAttachCGroupUDP6RecvmsgAttachCGroupGetsockoptAttachCGroupSetsockoptAttachTraceRawTpAttachTraceFEntryAttachTraceFExitAttachModifyReturnAttachLSMMacAttachTraceIter" +const _AttachType_name = "AttachNoneAttachCGroupInetEgressAttachCGroupInetSockCreateAttachCGroupSockOpsAttachSkSKBStreamParserAttachSkSKBStreamVerdictAttachCGroupDeviceAttachSkMsgVerdictAttachCGroupInet4BindAttachCGroupInet6BindAttachCGroupInet4ConnectAttachCGroupInet6ConnectAttachCGroupInet4PostBindAttachCGroupInet6PostBindAttachCGroupUDP4SendmsgAttachCGroupUDP6SendmsgAttachLircMode2AttachFlowDissectorAttachCGroupSysctlAttachCGroupUDP4RecvmsgAttachCGroupUDP6RecvmsgAttachCGroupGetsockoptAttachCGroupSetsockoptAttachTraceRawTpAttachTraceFEntryAttachTraceFExitAttachModifyReturnAttachLSMMacAttachTraceIterAttachCgroupInet4GetPeernameAttachCgroupInet6GetPeernameAttachCgroupInet4GetSocknameAttachCgroupInet6GetSocknameAttachXDPDevMapAttachCgroupInetSockReleaseAttachXDPCPUMapAttachSkLookupAttachXDP" -var _AttachType_index = [...]uint16{0, 10, 32, 58, 77, 100, 124, 142, 160, 181, 202, 226, 250, 275, 300, 323, 346, 361, 380, 398, 421, 444, 466, 488, 504, 521, 537, 555, 567, 582} +var _AttachType_index = [...]uint16{0, 10, 32, 58, 77, 100, 124, 142, 160, 181, 202, 226, 250, 275, 300, 323, 346, 361, 380, 398, 421, 444, 466, 488, 504, 521, 537, 555, 567, 582, 610, 638, 666, 694, 709, 736, 751, 765, 774} func (i AttachType) String() string { if i >= AttachType(len(_AttachType_index)-1) { @@ -135,3 +148,21 @@ func (i AttachType) String() string { } return _AttachType_name[_AttachType_index[i]:_AttachType_index[i+1]] } +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[PinNone-0] + _ = x[PinByName-1] +} + +const _PinType_name = "PinNonePinByName" + +var _PinType_index = [...]uint8{0, 7, 16} + +func (i PinType) String() string { + if i < 0 || i >= PinType(len(_PinType_index)-1) { + return "PinType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _PinType_name[_PinType_index[i]:_PinType_index[i+1]] +} diff --git a/vendor/github.com/containerd/cgroups/go.mod b/vendor/github.com/containerd/cgroups/go.mod index 64be56187e81d..eed71ffb485e4 100644 --- a/vendor/github.com/containerd/cgroups/go.mod +++ b/vendor/github.com/containerd/cgroups/go.mod @@ -3,16 +3,16 @@ module github.com/containerd/cgroups go 1.13 require ( - github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775 - github.com/coreos/go-systemd/v22 v22.0.0 + github.com/cilium/ebpf v0.4.0 + github.com/coreos/go-systemd/v22 v22.1.0 github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/docker/go-units v0.4.0 github.com/godbus/dbus/v5 v5.0.3 - github.com/gogo/protobuf v1.3.1 + github.com/gogo/protobuf v1.3.2 github.com/opencontainers/runtime-spec v1.0.2 github.com/pkg/errors v0.9.1 - github.com/sirupsen/logrus v1.6.0 - github.com/stretchr/testify v1.2.2 + github.com/sirupsen/logrus v1.7.0 + github.com/stretchr/testify v1.6.1 github.com/urfave/cli v1.22.2 - golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c ) diff --git a/vendor/github.com/containerd/cgroups/net_cls.go b/vendor/github.com/containerd/cgroups/net_cls.go index 28882575af5b0..839b06de080b1 100644 --- a/vendor/github.com/containerd/cgroups/net_cls.go +++ b/vendor/github.com/containerd/cgroups/net_cls.go @@ -55,3 +55,7 @@ func (n *netclsController) Create(path string, resources *specs.LinuxResources) } return nil } + +func (n *netclsController) Update(path string, resources *specs.LinuxResources) error { + return n.Create(path, resources) +} diff --git a/vendor/github.com/containerd/cgroups/stats/v1/metrics.pb.go b/vendor/github.com/containerd/cgroups/stats/v1/metrics.pb.go index a530f1d8fd4a8..6d2d41770b99c 100644 --- a/vendor/github.com/containerd/cgroups/stats/v1/metrics.pb.go +++ b/vendor/github.com/containerd/cgroups/stats/v1/metrics.pb.go @@ -9,6 +9,7 @@ import ( proto "github.com/gogo/protobuf/proto" io "io" math "math" + math_bits "math/bits" reflect "reflect" strings "strings" ) @@ -22,21 +23,21 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type Metrics struct { - Hugetlb []*HugetlbStat `protobuf:"bytes,1,rep,name=hugetlb,proto3" json:"hugetlb,omitempty"` - Pids *PidsStat `protobuf:"bytes,2,opt,name=pids,proto3" json:"pids,omitempty"` - CPU *CPUStat `protobuf:"bytes,3,opt,name=cpu,proto3" json:"cpu,omitempty"` - Memory *MemoryStat `protobuf:"bytes,4,opt,name=memory,proto3" json:"memory,omitempty"` - Blkio *BlkIOStat `protobuf:"bytes,5,opt,name=blkio,proto3" json:"blkio,omitempty"` - Rdma *RdmaStat `protobuf:"bytes,6,opt,name=rdma,proto3" json:"rdma,omitempty"` - Network []*NetworkStat `protobuf:"bytes,7,rep,name=network,proto3" json:"network,omitempty"` - CgroupStats *CgroupStats `protobuf:"bytes,8,opt,name=cgroup_stats,json=cgroupStats,proto3" json:"cgroup_stats,omitempty"` - MemoryOomControl *MemoryOomControl `protobuf:"bytes,9,opt,name=memory_oom_control,json=MemoryOomControl,proto3" json:"memory_oom_control,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Hugetlb []*HugetlbStat `protobuf:"bytes,1,rep,name=hugetlb,proto3" json:"hugetlb,omitempty"` + Pids *PidsStat `protobuf:"bytes,2,opt,name=pids,proto3" json:"pids,omitempty"` + CPU *CPUStat `protobuf:"bytes,3,opt,name=cpu,proto3" json:"cpu,omitempty"` + Memory *MemoryStat `protobuf:"bytes,4,opt,name=memory,proto3" json:"memory,omitempty"` + Blkio *BlkIOStat `protobuf:"bytes,5,opt,name=blkio,proto3" json:"blkio,omitempty"` + Rdma *RdmaStat `protobuf:"bytes,6,opt,name=rdma,proto3" json:"rdma,omitempty"` + Network []*NetworkStat `protobuf:"bytes,7,rep,name=network,proto3" json:"network,omitempty"` + CgroupStats *CgroupStats `protobuf:"bytes,8,opt,name=cgroup_stats,json=cgroupStats,proto3" json:"cgroup_stats,omitempty"` + MemoryOomControl *MemoryOomControl `protobuf:"bytes,9,opt,name=memory_oom_control,json=memoryOomControl,proto3" json:"memory_oom_control,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Metrics) Reset() { *m = Metrics{} } @@ -52,7 +53,7 @@ func (m *Metrics) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Metrics.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -94,7 +95,7 @@ func (m *HugetlbStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_HugetlbStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -134,7 +135,7 @@ func (m *PidsStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_PidsStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -174,7 +175,7 @@ func (m *CPUStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CPUStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -217,7 +218,7 @@ func (m *CPUUsage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CPUUsage.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -258,7 +259,7 @@ func (m *Throttle) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Throttle.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -332,7 +333,7 @@ func (m *MemoryStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MemoryStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -374,7 +375,7 @@ func (m *MemoryEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_MemoryEntry.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -394,15 +395,15 @@ func (m *MemoryEntry) XXX_DiscardUnknown() { var xxx_messageInfo_MemoryEntry proto.InternalMessageInfo type MemoryOomControl struct { - OomKillDisable uint64 `protobuf:"varint,1,opt,name=oom_kill_disable,proto3" json:oom_kill_disable",omitempty"` - UnderOom uint64 `protobuf:"varint,2,opt,name=under_oom,proto3" json:"under_oom,omitempty"` - OomKill uint64 `protobuf:"varint,3,opt,name=oom_kill,proto3" json:"oom_kill,omitempty"` + OomKillDisable uint64 `protobuf:"varint,1,opt,name=oom_kill_disable,json=oomKillDisable,proto3" json:"oom_kill_disable,omitempty"` + UnderOom uint64 `protobuf:"varint,2,opt,name=under_oom,json=underOom,proto3" json:"under_oom,omitempty"` + OomKill uint64 `protobuf:"varint,3,opt,name=oom_kill,json=oomKill,proto3" json:"oom_kill,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *MemoryOomControl) Reset() { *m = MemoryOomControl{} } +func (m *MemoryOomControl) Reset() { *m = MemoryOomControl{} } func (*MemoryOomControl) ProtoMessage() {} func (*MemoryOomControl) Descriptor() ([]byte, []int) { return fileDescriptor_a17b2d87c332bfaa, []int{8} @@ -415,7 +416,7 @@ func (m *MemoryOomControl) XXX_Marshal(b []byte, deterministic bool) ([]byte, er return xxx_messageInfo_MemoryOomControl.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -461,7 +462,7 @@ func (m *BlkIOStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BlkIOStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -504,7 +505,7 @@ func (m *BlkIOEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BlkIOEntry.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -544,7 +545,7 @@ func (m *RdmaStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RdmaStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -585,7 +586,7 @@ func (m *RdmaEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RdmaEntry.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -632,7 +633,7 @@ func (m *NetworkStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_NetworkStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -681,7 +682,7 @@ func (m *CgroupStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_CgroupStats.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -723,118 +724,123 @@ func init() { } var fileDescriptor_a17b2d87c332bfaa = []byte{ - // 1669 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x58, 0x4f, 0x73, 0x1b, 0xb7, - 0x15, 0x0f, 0xc5, 0x95, 0xc8, 0x7d, 0x94, 0x6c, 0x09, 0xfe, 0xb7, 0x52, 0x1c, 0x51, 0xa1, 0xec, - 0xd6, 0xad, 0xa7, 0xd2, 0x24, 0xed, 0x78, 0xea, 0x34, 0x99, 0x4e, 0xa4, 0x24, 0x63, 0x4f, 0xab, - 0x9a, 0x59, 0x4a, 0x93, 0xf6, 0xb4, 0x03, 0x2e, 0xe1, 0x25, 0xac, 0xe5, 0x62, 0x83, 0xc5, 0x52, - 0x74, 0x4f, 0x3d, 0x74, 0xa6, 0xa7, 0x7e, 0xa0, 0x7e, 0x83, 0x1c, 0x7b, 0xe9, 0x4c, 0x7b, 0xd1, - 0x34, 0xfc, 0x1c, 0x3d, 0x74, 0x80, 0x87, 0xfd, 0x43, 0xc7, 0xb2, 0xc2, 0xdb, 0xe2, 0xe1, 0xf7, - 0x7e, 0xef, 0xe1, 0xe1, 0x07, 0xe0, 0x91, 0xf0, 0xab, 0x88, 0xab, 0x71, 0x3e, 0x3c, 0x08, 0xc5, - 0xe4, 0x30, 0x14, 0x89, 0xa2, 0x3c, 0x61, 0x72, 0x74, 0x18, 0x46, 0x52, 0xe4, 0x69, 0x76, 0x98, - 0x29, 0xaa, 0xb2, 0xc3, 0xe9, 0x47, 0x87, 0x13, 0xa6, 0x24, 0x0f, 0xb3, 0x83, 0x54, 0x0a, 0x25, - 0x88, 0xc7, 0xc5, 0x41, 0x85, 0x3e, 0xb0, 0xe8, 0x83, 0xe9, 0x47, 0x3b, 0xb7, 0x23, 0x11, 0x09, - 0x03, 0x3a, 0xd4, 0x5f, 0x88, 0xef, 0xfd, 0xaf, 0x09, 0xad, 0x13, 0x64, 0x20, 0xbf, 0x85, 0xd6, - 0x38, 0x8f, 0x98, 0x8a, 0x87, 0x5e, 0x63, 0xaf, 0xf9, 0xa8, 0xf3, 0xf1, 0xc3, 0x83, 0xab, 0xd8, - 0x0e, 0x9e, 0x21, 0x70, 0xa0, 0xa8, 0xf2, 0x0b, 0x2f, 0xf2, 0x04, 0x9c, 0x94, 0x8f, 0x32, 0x6f, - 0x65, 0xaf, 0xf1, 0xa8, 0xf3, 0x71, 0xef, 0x6a, 0xef, 0x3e, 0x1f, 0x65, 0xc6, 0xd5, 0xe0, 0xc9, - 0xa7, 0xd0, 0x0c, 0xd3, 0xdc, 0x6b, 0x1a, 0xb7, 0x0f, 0xaf, 0x76, 0x3b, 0xee, 0x9f, 0x69, 0xaf, - 0xa3, 0xd6, 0xfc, 0xb2, 0xdb, 0x3c, 0xee, 0x9f, 0xf9, 0xda, 0x8d, 0x7c, 0x0a, 0x6b, 0x13, 0x36, - 0x11, 0xf2, 0xb5, 0xe7, 0x18, 0x82, 0x07, 0x57, 0x13, 0x9c, 0x18, 0x9c, 0x89, 0x6c, 0x7d, 0xc8, - 0x53, 0x58, 0x1d, 0xc6, 0xe7, 0x5c, 0x78, 0xab, 0xc6, 0x79, 0xff, 0x6a, 0xe7, 0xa3, 0xf8, 0xfc, - 0xf9, 0x0b, 0xe3, 0x8b, 0x1e, 0x7a, 0xb9, 0x72, 0x34, 0xa1, 0xde, 0xda, 0x75, 0xcb, 0xf5, 0x47, - 0x13, 0x8a, 0xcb, 0xd5, 0x78, 0x5d, 0xe7, 0x84, 0xa9, 0x0b, 0x21, 0xcf, 0xbd, 0xd6, 0x75, 0x75, - 0xfe, 0x03, 0x02, 0xb1, 0xce, 0xd6, 0x8b, 0x3c, 0x83, 0x75, 0x84, 0x04, 0x46, 0x05, 0x5e, 0xdb, - 0x24, 0xf0, 0x0e, 0x96, 0x63, 0xf3, 0xa9, 0x49, 0x32, 0xbf, 0x13, 0x56, 0x83, 0xde, 0x39, 0x74, - 0x6a, 0x3b, 0x49, 0x6e, 0xc3, 0x6a, 0x9e, 0xd1, 0x88, 0x79, 0x8d, 0xbd, 0xc6, 0x23, 0xc7, 0xc7, - 0x01, 0xd9, 0x84, 0xe6, 0x84, 0xce, 0xcc, 0xae, 0x3a, 0xbe, 0xfe, 0x24, 0x1e, 0xb4, 0x5e, 0x52, - 0x1e, 0x87, 0x89, 0x32, 0x9b, 0xe6, 0xf8, 0xc5, 0x90, 0xec, 0x40, 0x3b, 0xa5, 0x11, 0xcb, 0xf8, - 0x9f, 0x99, 0xd9, 0x0e, 0xd7, 0x2f, 0xc7, 0xbd, 0x4f, 0xa0, 0x5d, 0x6c, 0xbc, 0x66, 0x08, 0x73, - 0x29, 0x59, 0xa2, 0x6c, 0xac, 0x62, 0xa8, 0x73, 0x88, 0xf9, 0x84, 0x2b, 0x1b, 0x0f, 0x07, 0xbd, - 0xbf, 0x35, 0xa0, 0x65, 0xb7, 0x9f, 0xfc, 0xba, 0x9e, 0xe5, 0x3b, 0x0b, 0x7f, 0xdc, 0x3f, 0x3b, - 0xd3, 0xc8, 0x62, 0x25, 0x47, 0x00, 0x6a, 0x2c, 0x85, 0x52, 0x31, 0x4f, 0xa2, 0xeb, 0x65, 0x7a, - 0x8a, 0x58, 0xe6, 0xd7, 0xbc, 0x7a, 0xdf, 0x42, 0xbb, 0xa0, 0xd5, 0xb9, 0x2a, 0xa1, 0x68, 0x5c, - 0xd4, 0xcb, 0x0c, 0xc8, 0x5d, 0x58, 0x3b, 0x67, 0x32, 0x61, 0xb1, 0x5d, 0x82, 0x1d, 0x11, 0x02, - 0x4e, 0x9e, 0x31, 0x69, 0x4b, 0x66, 0xbe, 0xc9, 0x3e, 0xb4, 0x52, 0x26, 0x03, 0x2d, 0x7f, 0x67, - 0xaf, 0xf9, 0xc8, 0x39, 0x82, 0xf9, 0x65, 0x77, 0xad, 0xcf, 0xa4, 0x96, 0xf7, 0x5a, 0xca, 0xe4, - 0x71, 0x9a, 0xf7, 0x66, 0xd0, 0x2e, 0x52, 0xd1, 0x85, 0x4b, 0x99, 0xe4, 0x62, 0x94, 0x15, 0x85, - 0xb3, 0x43, 0xf2, 0x18, 0xb6, 0x6c, 0x9a, 0x6c, 0x14, 0x14, 0x18, 0xcc, 0x60, 0xb3, 0x9c, 0xe8, - 0x5b, 0xf0, 0x43, 0xb8, 0x51, 0x81, 0x15, 0x9f, 0x30, 0x9b, 0xd5, 0x46, 0x69, 0x3d, 0xe5, 0x13, - 0xd6, 0xfb, 0x4f, 0x07, 0xa0, 0x3a, 0x34, 0x7a, 0xbd, 0x21, 0x0d, 0xc7, 0xa5, 0x3e, 0xcc, 0x80, - 0x6c, 0x43, 0x53, 0x66, 0x36, 0x14, 0x9e, 0x4d, 0x7f, 0x30, 0xf0, 0xb5, 0x8d, 0xfc, 0x04, 0xda, - 0x32, 0xcb, 0x02, 0x7d, 0x41, 0x60, 0x80, 0xa3, 0xce, 0xfc, 0xb2, 0xdb, 0xf2, 0x07, 0x03, 0x2d, - 0x3b, 0xbf, 0x25, 0xb3, 0x4c, 0x7f, 0x90, 0x2e, 0x74, 0x26, 0x34, 0x4d, 0xd9, 0x28, 0x78, 0xc9, - 0x63, 0x54, 0x8e, 0xe3, 0x03, 0x9a, 0xbe, 0xe2, 0xb1, 0xa9, 0xf4, 0x88, 0x4b, 0xf5, 0xda, 0x1c, - 0x53, 0xc7, 0xc7, 0x01, 0xb9, 0x0f, 0xee, 0x85, 0xe4, 0x8a, 0x0d, 0x69, 0x78, 0x6e, 0x8e, 0xa1, - 0xe3, 0x57, 0x06, 0xe2, 0x41, 0x3b, 0x8d, 0x82, 0x34, 0x0a, 0x78, 0xe2, 0xb5, 0x70, 0x27, 0xd2, - 0xa8, 0x1f, 0x3d, 0x4f, 0xc8, 0x0e, 0xb8, 0x38, 0x23, 0x72, 0x65, 0x4e, 0x8f, 0x2e, 0x63, 0xd4, - 0x8f, 0x5e, 0xe4, 0x8a, 0x6c, 0x1b, 0xaf, 0x97, 0x34, 0x8f, 0x95, 0xe7, 0x16, 0x53, 0x5f, 0xe9, - 0x21, 0xd9, 0x83, 0xf5, 0x34, 0x0a, 0x26, 0xf4, 0x95, 0x9d, 0x06, 0x4c, 0x33, 0x8d, 0x4e, 0xe8, - 0x2b, 0x44, 0xec, 0xc3, 0x06, 0x4f, 0x68, 0xa8, 0xf8, 0x94, 0x05, 0x34, 0x11, 0x89, 0xd7, 0x31, - 0x90, 0xf5, 0xc2, 0xf8, 0x79, 0x22, 0x12, 0xbd, 0xd8, 0x3a, 0x64, 0x1d, 0x59, 0x6a, 0x80, 0x3a, - 0x8b, 0xa9, 0xc7, 0xc6, 0x22, 0x8b, 0xa9, 0x48, 0xc5, 0x62, 0x20, 0x37, 0xea, 0x2c, 0x06, 0xb0, - 0x07, 0x9d, 0x3c, 0x61, 0x53, 0x1e, 0x2a, 0x3a, 0x8c, 0x99, 0x77, 0xd3, 0x00, 0xea, 0x26, 0xf2, - 0x09, 0x6c, 0x8f, 0x39, 0x93, 0x54, 0x86, 0x63, 0x1e, 0xd2, 0x38, 0xc0, 0x2b, 0x31, 0xc0, 0xe3, - 0xb7, 0x69, 0xf0, 0xf7, 0xea, 0x00, 0x54, 0xc2, 0xef, 0xf5, 0x34, 0x79, 0x02, 0x0b, 0x53, 0x41, - 0x76, 0x41, 0x53, 0xeb, 0xb9, 0x65, 0x3c, 0xef, 0xd4, 0xa7, 0x07, 0x17, 0x34, 0x45, 0xbf, 0x2e, - 0x74, 0xcc, 0x29, 0x09, 0x50, 0x48, 0x04, 0xd3, 0x36, 0xa6, 0x63, 0xa3, 0xa6, 0x9f, 0x81, 0x8b, - 0x00, 0xad, 0xa9, 0x5b, 0x46, 0x33, 0xeb, 0xf3, 0xcb, 0x6e, 0xfb, 0x54, 0x1b, 0xb5, 0xb0, 0xda, - 0x66, 0xda, 0xcf, 0x32, 0xf2, 0x04, 0x6e, 0x94, 0x50, 0xd4, 0xd8, 0x6d, 0x83, 0xdf, 0x9c, 0x5f, - 0x76, 0xd7, 0x0b, 0xbc, 0x11, 0xda, 0x7a, 0xe1, 0x63, 0xd4, 0xf6, 0x73, 0xd8, 0x42, 0xbf, 0xba, - 0xe6, 0xee, 0x98, 0x4c, 0x6e, 0x9a, 0x89, 0x93, 0x4a, 0x78, 0x65, 0xbe, 0x28, 0xbf, 0xbb, 0xb5, - 0x7c, 0xbf, 0x30, 0x1a, 0xfc, 0x29, 0xa0, 0x4f, 0x50, 0x29, 0xf1, 0x9e, 0x01, 0x61, 0x6e, 0xdf, - 0x94, 0x72, 0xdc, 0x2f, 0xb2, 0x2d, 0x45, 0xe9, 0xe1, 0x96, 0x18, 0x6b, 0x1f, 0x95, 0xf9, 0xb0, - 0x60, 0xab, 0xf4, 0xb9, 0x8d, 0x9b, 0x5f, 0xa2, 0xb4, 0x48, 0x1f, 0xd4, 0xb8, 0x50, 0x8b, 0x3b, - 0x0b, 0x28, 0x54, 0xe3, 0x63, 0x20, 0x25, 0xaa, 0x52, 0xed, 0xfb, 0xb5, 0x85, 0xf6, 0x2b, 0xe9, - 0x1e, 0xc0, 0x2d, 0x04, 0x2f, 0x0a, 0xf8, 0xbe, 0x41, 0x63, 0xbd, 0x9e, 0xd7, 0x55, 0x5c, 0x16, - 0xb1, 0x8e, 0xfe, 0xa0, 0xc6, 0xfd, 0x79, 0x85, 0xfd, 0x21, 0xb7, 0x29, 0xf9, 0xee, 0x5b, 0xb8, - 0x4d, 0xd1, 0xdf, 0xe4, 0x36, 0xe8, 0xee, 0x0f, 0xb8, 0x0d, 0xf6, 0x71, 0x81, 0xad, 0x8b, 0x7d, - 0xcf, 0x5e, 0x7b, 0x7a, 0xe2, 0xac, 0xa6, 0xf8, 0xdf, 0x14, 0x4f, 0xc7, 0x87, 0xd7, 0x3d, 0x99, - 0xa8, 0xf5, 0x2f, 0x13, 0x25, 0x5f, 0x17, 0xaf, 0xc7, 0x53, 0x70, 0xb4, 0xca, 0xbd, 0xde, 0x32, - 0xbe, 0xc6, 0x85, 0x7c, 0x56, 0x3e, 0x09, 0xfb, 0xcb, 0x38, 0x17, 0x2f, 0xc7, 0x00, 0x00, 0xbf, - 0x02, 0x15, 0xa6, 0xde, 0x83, 0x25, 0x28, 0x8e, 0x36, 0xe6, 0x97, 0x5d, 0xf7, 0x77, 0xc6, 0xf9, - 0xf4, 0xb8, 0xef, 0xbb, 0xc8, 0x73, 0x1a, 0xa6, 0x3d, 0x06, 0x9d, 0x1a, 0xb0, 0x7a, 0x77, 0x1b, - 0xb5, 0x77, 0xb7, 0xea, 0x08, 0x56, 0xde, 0xd2, 0x11, 0x34, 0xdf, 0xda, 0x11, 0x38, 0x0b, 0x1d, - 0x41, 0xef, 0x5f, 0xab, 0xe0, 0x96, 0xad, 0x13, 0xa1, 0xb0, 0xc3, 0x45, 0x90, 0x31, 0x39, 0xe5, - 0x21, 0x0b, 0x86, 0xaf, 0x15, 0xcb, 0x02, 0xc9, 0xc2, 0x5c, 0x66, 0x7c, 0xca, 0x6c, 0xdb, 0xf9, - 0xe0, 0x9a, 0x1e, 0x0c, 0x6b, 0x73, 0x8f, 0x8b, 0x01, 0xd2, 0x1c, 0x69, 0x16, 0xbf, 0x20, 0x21, - 0x7f, 0x84, 0x3b, 0x55, 0x88, 0x51, 0x8d, 0x7d, 0x65, 0x09, 0xf6, 0x5b, 0x25, 0xfb, 0xa8, 0x62, - 0x3e, 0x85, 0x5b, 0x5c, 0x04, 0xdf, 0xe6, 0x2c, 0x5f, 0xe0, 0x6d, 0x2e, 0xc1, 0xbb, 0xc5, 0xc5, - 0xd7, 0xc6, 0xbf, 0x62, 0x0d, 0x60, 0xbb, 0x56, 0x12, 0xfd, 0x16, 0xd7, 0xb8, 0x9d, 0x25, 0xb8, - 0xef, 0x96, 0x39, 0xeb, 0xb7, 0xbb, 0x0a, 0xf0, 0x27, 0xb8, 0xcb, 0x45, 0x70, 0x41, 0xb9, 0x7a, - 0x93, 0x7d, 0x75, 0xb9, 0x8a, 0x7c, 0x43, 0xb9, 0x5a, 0xa4, 0xc6, 0x8a, 0x4c, 0x98, 0x8c, 0x16, - 0x2a, 0xb2, 0xb6, 0x5c, 0x45, 0x4e, 0x8c, 0x7f, 0xc5, 0xda, 0x87, 0x2d, 0x2e, 0xde, 0xcc, 0xb5, - 0xb5, 0x04, 0xe7, 0x4d, 0x2e, 0x16, 0xf3, 0xfc, 0x1a, 0xb6, 0x32, 0x16, 0x2a, 0x21, 0xeb, 0x6a, - 0x6b, 0x2f, 0xc1, 0xb8, 0x69, 0xdd, 0x4b, 0xca, 0xde, 0x14, 0xa0, 0x9a, 0x27, 0x37, 0x60, 0x45, - 0xa4, 0xe6, 0xe8, 0xb8, 0xfe, 0x8a, 0x48, 0x75, 0x0f, 0x38, 0xd2, 0xd7, 0x0e, 0x1e, 0x1c, 0xd7, - 0xb7, 0x23, 0x7d, 0x9e, 0x26, 0xf4, 0x95, 0x28, 0x9a, 0x40, 0x1c, 0x18, 0x2b, 0x4f, 0x84, 0xb4, - 0x67, 0x07, 0x07, 0xda, 0x3a, 0xa5, 0x71, 0xce, 0x8a, 0x9e, 0xc7, 0x0c, 0x7a, 0x7f, 0x6d, 0x40, - 0xbb, 0xf8, 0x41, 0x41, 0x3e, 0xab, 0xb7, 0xd1, 0xcd, 0x77, 0xff, 0x7e, 0xd1, 0x4e, 0xb8, 0x98, - 0xb2, 0xd7, 0x7e, 0x5a, 0xf5, 0xda, 0x3f, 0xda, 0xd9, 0x36, 0xe4, 0x0c, 0xdc, 0xd2, 0x56, 0x5b, - 0x6d, 0x63, 0x61, 0xb5, 0x5d, 0xe8, 0x8c, 0x43, 0x1a, 0x8c, 0x69, 0x32, 0x8a, 0x19, 0x76, 0x88, - 0x1b, 0x3e, 0x8c, 0x43, 0xfa, 0x0c, 0x2d, 0x05, 0x40, 0x0c, 0x5f, 0xb1, 0x50, 0x65, 0xa6, 0x28, - 0x08, 0x78, 0x81, 0x96, 0xde, 0xdf, 0x57, 0xa0, 0x53, 0xfb, 0x0d, 0xa4, 0x7b, 0xe8, 0x84, 0x4e, - 0x8a, 0x38, 0xe6, 0x5b, 0x77, 0x6c, 0x72, 0x86, 0x77, 0x89, 0xbd, 0xa6, 0x5a, 0x72, 0x66, 0x2e, - 0x05, 0xf2, 0x01, 0x80, 0x9c, 0x05, 0x29, 0x0d, 0xcf, 0x99, 0xa5, 0x77, 0x7c, 0x57, 0xce, 0xfa, - 0x68, 0x20, 0xef, 0x83, 0x2b, 0x67, 0x01, 0x93, 0x52, 0xc8, 0xcc, 0xd6, 0xbe, 0x2d, 0x67, 0x5f, - 0x9a, 0xb1, 0xf5, 0x1d, 0x49, 0xa1, 0x7b, 0x01, 0xbb, 0x07, 0xae, 0x9c, 0x7d, 0x81, 0x06, 0x1d, - 0x55, 0x15, 0x51, 0xb1, 0xf5, 0x6c, 0xa9, 0x2a, 0xaa, 0xaa, 0xa2, 0x62, 0xeb, 0xe9, 0xaa, 0x7a, - 0x54, 0x55, 0x46, 0xc5, 0xee, 0xb3, 0xad, 0x6a, 0x51, 0x55, 0x15, 0xd5, 0x2d, 0x7c, 0x6d, 0xd4, - 0xde, 0x3f, 0x1a, 0xd0, 0xa9, 0xfd, 0x9a, 0xd3, 0x05, 0x4c, 0x64, 0x90, 0xc5, 0x8c, 0xa5, 0xfa, - 0x27, 0x0d, 0xde, 0xdd, 0x90, 0xc8, 0x81, 0xb5, 0x68, 0xbe, 0x44, 0x06, 0x32, 0x4f, 0x92, 0xe2, - 0x27, 0x8f, 0xe3, 0xbb, 0x89, 0xf4, 0xd1, 0x60, 0xa7, 0x33, 0x85, 0xe1, 0x9a, 0xc5, 0xf4, 0x00, - 0x0d, 0xe4, 0x17, 0x40, 0x12, 0x19, 0xe4, 0x09, 0x4f, 0x14, 0x93, 0x32, 0x4f, 0x15, 0x1f, 0x96, - 0xed, 0xf9, 0x56, 0x22, 0xcf, 0x16, 0x27, 0xc8, 0x7d, 0xc3, 0x66, 0x2f, 0x1b, 0x5b, 0xb2, 0x76, - 0x22, 0x9f, 0x9b, 0x9b, 0xe3, 0xc8, 0xfb, 0xee, 0xfb, 0xdd, 0xf7, 0xfe, 0xfd, 0xfd, 0xee, 0x7b, - 0x7f, 0x99, 0xef, 0x36, 0xbe, 0x9b, 0xef, 0x36, 0xfe, 0x39, 0xdf, 0x6d, 0xfc, 0x77, 0xbe, 0xdb, - 0x18, 0xae, 0x99, 0x3f, 0x23, 0x7e, 0xf9, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb9, 0x78, 0x66, - 0x06, 0xf4, 0x10, 0x00, 0x00, + // 1749 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x58, 0xcd, 0x72, 0xe3, 0xc6, + 0x11, 0x36, 0x45, 0x48, 0x24, 0x9a, 0x92, 0x56, 0x9a, 0xfd, 0x83, 0xe4, 0xb5, 0x28, 0x53, 0xbb, + 0x89, 0xe2, 0xad, 0x48, 0x65, 0x27, 0xb5, 0x95, 0x75, 0xec, 0x4a, 0x59, 0x5a, 0xbb, 0x76, 0xcb, + 0x51, 0x44, 0x83, 0x52, 0xd9, 0x39, 0xa1, 0x40, 0x70, 0x16, 0x9c, 0x15, 0x80, 0x81, 0x07, 0x03, + 0x89, 0xca, 0x29, 0x87, 0x54, 0xe5, 0x94, 0x07, 0xca, 0x1b, 0xf8, 0x98, 0x4b, 0x52, 0xc9, 0x45, + 0x15, 0xf3, 0x49, 0x52, 0x33, 0x3d, 0xf8, 0xa1, 0xbc, 0x5a, 0x85, 0x37, 0x76, 0xcf, 0xd7, 0x5f, + 0xf7, 0x34, 0xbe, 0x19, 0x34, 0x08, 0xbf, 0x0e, 0x99, 0x1c, 0xe7, 0xc3, 0xbd, 0x80, 0xc7, 0xfb, + 0x01, 0x4f, 0xa4, 0xcf, 0x12, 0x2a, 0x46, 0xfb, 0x41, 0x28, 0x78, 0x9e, 0x66, 0xfb, 0x99, 0xf4, + 0x65, 0xb6, 0x7f, 0xfe, 0xf1, 0x7e, 0x4c, 0xa5, 0x60, 0x41, 0xb6, 0x97, 0x0a, 0x2e, 0x39, 0x71, + 0x18, 0xdf, 0xab, 0xd0, 0x7b, 0x06, 0xbd, 0x77, 0xfe, 0xf1, 0xe6, 0xbd, 0x90, 0x87, 0x5c, 0x83, + 0xf6, 0xd5, 0x2f, 0xc4, 0xf7, 0xfe, 0x65, 0x41, 0xeb, 0x08, 0x19, 0xc8, 0xef, 0xa0, 0x35, 0xce, + 0x43, 0x2a, 0xa3, 0xa1, 0xd3, 0xd8, 0x6e, 0xee, 0x76, 0x3e, 0x79, 0xb2, 0x77, 0x13, 0xdb, 0xde, + 0x4b, 0x04, 0x0e, 0xa4, 0x2f, 0xdd, 0x22, 0x8a, 0x3c, 0x03, 0x2b, 0x65, 0xa3, 0xcc, 0x59, 0xd8, + 0x6e, 0xec, 0x76, 0x3e, 0xe9, 0xdd, 0x1c, 0xdd, 0x67, 0xa3, 0x4c, 0x87, 0x6a, 0x3c, 0xf9, 0x0c, + 0x9a, 0x41, 0x9a, 0x3b, 0x4d, 0x1d, 0xf6, 0xe1, 0xcd, 0x61, 0x87, 0xfd, 0x53, 0x15, 0x75, 0xd0, + 0x9a, 0x5e, 0x75, 0x9b, 0x87, 0xfd, 0x53, 0x57, 0x85, 0x91, 0xcf, 0x60, 0x29, 0xa6, 0x31, 0x17, + 0x97, 0x8e, 0xa5, 0x09, 0x1e, 0xdf, 0x4c, 0x70, 0xa4, 0x71, 0x3a, 0xb3, 0x89, 0x21, 0xcf, 0x61, + 0x71, 0x18, 0x9d, 0x31, 0xee, 0x2c, 0xea, 0xe0, 0x9d, 0x9b, 0x83, 0x0f, 0xa2, 0xb3, 0x57, 0xc7, + 0x3a, 0x16, 0x23, 0xd4, 0x76, 0xc5, 0x28, 0xf6, 0x9d, 0xa5, 0xdb, 0xb6, 0xeb, 0x8e, 0x62, 0x1f, + 0xb7, 0xab, 0xf0, 0xaa, 0xcf, 0x09, 0x95, 0x17, 0x5c, 0x9c, 0x39, 0xad, 0xdb, 0xfa, 0xfc, 0x07, + 0x04, 0x62, 0x9f, 0x4d, 0x14, 0x79, 0x09, 0xcb, 0x08, 0xf1, 0xb4, 0x0a, 0x9c, 0xb6, 0x2e, 0xe0, + 0x1d, 0x2c, 0x87, 0xfa, 0xa7, 0x22, 0xc9, 0xdc, 0x4e, 0x50, 0x19, 0xe4, 0x3b, 0x20, 0xd8, 0x07, + 0x8f, 0xf3, 0xd8, 0x53, 0xc1, 0x82, 0x47, 0x8e, 0xad, 0xf9, 0x3e, 0xba, 0xad, 0x8f, 0xc7, 0x3c, + 0x3e, 0xc4, 0x08, 0x77, 0x2d, 0xbe, 0xe6, 0xe9, 0x9d, 0x41, 0xa7, 0xa6, 0x11, 0x72, 0x0f, 0x16, + 0xf3, 0xcc, 0x0f, 0xa9, 0xd3, 0xd8, 0x6e, 0xec, 0x5a, 0x2e, 0x1a, 0x64, 0x0d, 0x9a, 0xb1, 0x3f, + 0xd1, 0x7a, 0xb1, 0x5c, 0xf5, 0x93, 0x38, 0xd0, 0x7a, 0xed, 0xb3, 0x28, 0x48, 0xa4, 0x96, 0x83, + 0xe5, 0x16, 0x26, 0xd9, 0x84, 0x76, 0xea, 0x87, 0x34, 0x63, 0x7f, 0xa2, 0xfa, 0x41, 0xdb, 0x6e, + 0x69, 0xf7, 0x3e, 0x85, 0x76, 0x21, 0x29, 0xc5, 0x10, 0xe4, 0x42, 0xd0, 0x44, 0x9a, 0x5c, 0x85, + 0xa9, 0x6a, 0x88, 0x58, 0xcc, 0xa4, 0xc9, 0x87, 0x46, 0xef, 0xaf, 0x0d, 0x68, 0x19, 0x61, 0x91, + 0xdf, 0xd4, 0xab, 0x7c, 0xe7, 0x23, 0x3d, 0xec, 0x9f, 0x9e, 0x2a, 0x64, 0xb1, 0x93, 0x03, 0x00, + 0x39, 0x16, 0x5c, 0xca, 0x88, 0x25, 0xe1, 0xed, 0x07, 0xe0, 0x04, 0xb1, 0xd4, 0xad, 0x45, 0xf5, + 0xbe, 0x87, 0x76, 0x41, 0xab, 0x6a, 0x95, 0x5c, 0xfa, 0x51, 0xd1, 0x2f, 0x6d, 0x90, 0x07, 0xb0, + 0x74, 0x46, 0x45, 0x42, 0x23, 0xb3, 0x05, 0x63, 0x11, 0x02, 0x56, 0x9e, 0x51, 0x61, 0x5a, 0xa6, + 0x7f, 0x93, 0x1d, 0x68, 0xa5, 0x54, 0x78, 0xea, 0x60, 0x59, 0xdb, 0xcd, 0x5d, 0xeb, 0x00, 0xa6, + 0x57, 0xdd, 0xa5, 0x3e, 0x15, 0xea, 0xe0, 0x2c, 0xa5, 0x54, 0x1c, 0xa6, 0x79, 0x6f, 0x02, 0xed, + 0xa2, 0x14, 0xd5, 0xb8, 0x94, 0x0a, 0xc6, 0x47, 0x59, 0xd1, 0x38, 0x63, 0x92, 0xa7, 0xb0, 0x6e, + 0xca, 0xa4, 0x23, 0xaf, 0xc0, 0x60, 0x05, 0x6b, 0xe5, 0x42, 0xdf, 0x80, 0x9f, 0xc0, 0x6a, 0x05, + 0x96, 0x2c, 0xa6, 0xa6, 0xaa, 0x95, 0xd2, 0x7b, 0xc2, 0x62, 0xda, 0xfb, 0x4f, 0x07, 0xa0, 0x3a, + 0x8e, 0x6a, 0xbf, 0x81, 0x1f, 0x8c, 0x4b, 0x7d, 0x68, 0x83, 0x6c, 0x40, 0x53, 0x64, 0x26, 0x15, + 0x9e, 0x7a, 0x77, 0x30, 0x70, 0x95, 0x8f, 0xfc, 0x0c, 0xda, 0x22, 0xcb, 0x3c, 0x75, 0xf5, 0x60, + 0x82, 0x83, 0xce, 0xf4, 0xaa, 0xdb, 0x72, 0x07, 0x03, 0x25, 0x3b, 0xb7, 0x25, 0xb2, 0x4c, 0xfd, + 0x20, 0x5d, 0xe8, 0xc4, 0x7e, 0x9a, 0xd2, 0x91, 0xf7, 0x9a, 0x45, 0xa8, 0x1c, 0xcb, 0x05, 0x74, + 0x7d, 0xc5, 0x22, 0xdd, 0xe9, 0x11, 0x13, 0xf2, 0x52, 0x5f, 0x00, 0x96, 0x8b, 0x06, 0x79, 0x04, + 0xf6, 0x85, 0x60, 0x92, 0x0e, 0xfd, 0xe0, 0x4c, 0x1f, 0x70, 0xcb, 0xad, 0x1c, 0xc4, 0x81, 0x76, + 0x1a, 0x7a, 0x69, 0xe8, 0xb1, 0xc4, 0x69, 0xe1, 0x93, 0x48, 0xc3, 0x7e, 0xf8, 0x2a, 0x21, 0x9b, + 0x60, 0xe3, 0x0a, 0xcf, 0xa5, 0x3e, 0x97, 0xaa, 0x8d, 0x61, 0x3f, 0x3c, 0xce, 0x25, 0xd9, 0xd0, + 0x51, 0xaf, 0xfd, 0x3c, 0x92, 0xfa, 0x88, 0xe9, 0xa5, 0xaf, 0x94, 0x49, 0xb6, 0x61, 0x39, 0x0d, + 0xbd, 0xd8, 0x7f, 0x63, 0x96, 0x01, 0xcb, 0x4c, 0xc3, 0x23, 0xff, 0x0d, 0x22, 0x76, 0x60, 0x85, + 0x25, 0x7e, 0x20, 0xd9, 0x39, 0xf5, 0xfc, 0x84, 0x27, 0x4e, 0x47, 0x43, 0x96, 0x0b, 0xe7, 0x17, + 0x09, 0x4f, 0xd4, 0x66, 0xeb, 0x90, 0x65, 0x64, 0xa9, 0x01, 0xea, 0x2c, 0xba, 0x1f, 0x2b, 0xb3, + 0x2c, 0xba, 0x23, 0x15, 0x8b, 0x86, 0xac, 0xd6, 0x59, 0x34, 0x60, 0x1b, 0x3a, 0x79, 0x42, 0xcf, + 0x59, 0x20, 0xfd, 0x61, 0x44, 0x9d, 0x3b, 0x1a, 0x50, 0x77, 0x91, 0x4f, 0x61, 0x63, 0xcc, 0xa8, + 0xf0, 0x45, 0x30, 0x66, 0x81, 0x1f, 0x79, 0xe6, 0x92, 0xc1, 0xe3, 0xb7, 0xa6, 0xf1, 0x0f, 0xeb, + 0x00, 0x54, 0xc2, 0xef, 0xd5, 0x32, 0x79, 0x06, 0x33, 0x4b, 0x5e, 0x76, 0xe1, 0xa7, 0x26, 0x72, + 0x5d, 0x47, 0xde, 0xaf, 0x2f, 0x0f, 0x2e, 0xfc, 0x14, 0xe3, 0xba, 0xd0, 0xd1, 0xa7, 0xc4, 0x43, + 0x21, 0x11, 0x2c, 0x5b, 0xbb, 0x0e, 0xb5, 0x9a, 0x7e, 0x01, 0x36, 0x02, 0x94, 0xa6, 0xee, 0x6a, + 0xcd, 0x2c, 0x4f, 0xaf, 0xba, 0xed, 0x13, 0xe5, 0x54, 0xc2, 0x6a, 0xeb, 0x65, 0x37, 0xcb, 0xc8, + 0x33, 0x58, 0x2d, 0xa1, 0xa8, 0xb1, 0x7b, 0x1a, 0xbf, 0x36, 0xbd, 0xea, 0x2e, 0x17, 0x78, 0x2d, + 0xb4, 0xe5, 0x22, 0x46, 0xab, 0xed, 0x23, 0x58, 0xc7, 0xb8, 0xba, 0xe6, 0xee, 0xeb, 0x4a, 0xee, + 0xe8, 0x85, 0xa3, 0x4a, 0x78, 0x65, 0xbd, 0x28, 0xbf, 0x07, 0xb5, 0x7a, 0x5f, 0x68, 0x0d, 0xfe, + 0x1c, 0x30, 0xc6, 0xab, 0x94, 0xf8, 0x50, 0x83, 0xb0, 0xb6, 0x6f, 0x4b, 0x39, 0xee, 0x14, 0xd5, + 0x96, 0xa2, 0x74, 0xf0, 0x91, 0x68, 0x6f, 0x1f, 0x95, 0xf9, 0xa4, 0x60, 0xab, 0xf4, 0xb9, 0x81, + 0x0f, 0xbf, 0x44, 0x29, 0x91, 0x3e, 0xae, 0x71, 0xa1, 0x16, 0x37, 0x67, 0x50, 0xa8, 0xc6, 0xa7, + 0x40, 0x4a, 0x54, 0xa5, 0xda, 0xf7, 0x6b, 0x1b, 0xed, 0x57, 0xd2, 0xdd, 0x83, 0xbb, 0x08, 0x9e, + 0x15, 0xf0, 0x23, 0x8d, 0xc6, 0x7e, 0xbd, 0xaa, 0xab, 0xb8, 0x6c, 0x62, 0x1d, 0xfd, 0x41, 0x8d, + 0xfb, 0x8b, 0x0a, 0xfb, 0x53, 0x6e, 0xdd, 0xf2, 0xad, 0xb7, 0x70, 0xeb, 0xa6, 0x5f, 0xe7, 0xd6, + 0xe8, 0xee, 0x4f, 0xb8, 0x35, 0xf6, 0x69, 0x81, 0xad, 0x8b, 0x7d, 0xdb, 0x5c, 0x7b, 0x6a, 0xe1, + 0xb4, 0xa6, 0xf8, 0xdf, 0x16, 0xaf, 0x8e, 0x0f, 0x6f, 0x7b, 0x19, 0xa3, 0xd6, 0xbf, 0x4c, 0xa4, + 0xb8, 0x2c, 0xde, 0x1e, 0xcf, 0xc1, 0x52, 0x2a, 0x77, 0x7a, 0xf3, 0xc4, 0xea, 0x10, 0xf2, 0x79, + 0xf9, 0x4a, 0xd8, 0x99, 0x27, 0xb8, 0x78, 0x73, 0x0c, 0x00, 0xf0, 0x97, 0x27, 0x83, 0xd4, 0x79, + 0x3c, 0x07, 0xc5, 0xc1, 0xca, 0xf4, 0xaa, 0x6b, 0x7f, 0xad, 0x83, 0x4f, 0x0e, 0xfb, 0xae, 0x8d, + 0x3c, 0x27, 0x41, 0xda, 0xa3, 0xd0, 0xa9, 0x01, 0xab, 0xf7, 0x6e, 0xa3, 0xf6, 0xde, 0xad, 0x26, + 0x82, 0x85, 0xb7, 0x4c, 0x04, 0xcd, 0xb7, 0x4e, 0x04, 0xd6, 0xcc, 0x44, 0xd0, 0x93, 0xb0, 0x76, + 0x7d, 0x10, 0x21, 0xbb, 0xb0, 0xa6, 0x26, 0x99, 0x33, 0x16, 0xa9, 0x73, 0x95, 0xe9, 0x47, 0x86, + 0x69, 0x57, 0x39, 0x8f, 0xbf, 0x66, 0x51, 0xf4, 0x02, 0xbd, 0xe4, 0x7d, 0xb0, 0xf3, 0x64, 0x44, + 0x85, 0x9a, 0x7c, 0x4c, 0x0d, 0x6d, 0xed, 0x38, 0xe6, 0xb1, 0xba, 0xaa, 0x0b, 0x9a, 0x62, 0x0e, + 0x31, 0xe1, 0xbd, 0x7f, 0x2e, 0x82, 0x5d, 0x8e, 0x82, 0xc4, 0x87, 0x4d, 0xc6, 0xbd, 0x8c, 0x8a, + 0x73, 0x16, 0x50, 0x6f, 0x78, 0x29, 0x69, 0xe6, 0x09, 0x1a, 0xe4, 0x22, 0x63, 0xe7, 0xd4, 0x8c, + 0xd1, 0x8f, 0x6f, 0x99, 0x29, 0xf1, 0x89, 0x3c, 0x64, 0x7c, 0x80, 0x34, 0x07, 0x8a, 0xc5, 0x2d, + 0x48, 0xc8, 0x77, 0x70, 0xbf, 0x4a, 0x31, 0xaa, 0xb1, 0x2f, 0xcc, 0xc1, 0x7e, 0xb7, 0x64, 0x1f, + 0x55, 0xcc, 0x27, 0x70, 0x97, 0x71, 0xef, 0xfb, 0x9c, 0xe6, 0x33, 0xbc, 0xcd, 0x39, 0x78, 0xd7, + 0x19, 0xff, 0x46, 0xc7, 0x57, 0xac, 0x1e, 0x6c, 0xd4, 0x5a, 0xa2, 0x26, 0x80, 0x1a, 0xb7, 0x35, + 0x07, 0xf7, 0x83, 0xb2, 0x66, 0x35, 0x31, 0x54, 0x09, 0xfe, 0x08, 0x0f, 0x18, 0xf7, 0x2e, 0x7c, + 0x26, 0xaf, 0xb3, 0x2f, 0xce, 0xd7, 0x91, 0x6f, 0x7d, 0x26, 0x67, 0xa9, 0xb1, 0x23, 0x31, 0x15, + 0xe1, 0x4c, 0x47, 0x96, 0xe6, 0xeb, 0xc8, 0x91, 0x8e, 0xaf, 0x58, 0xfb, 0xb0, 0xce, 0xf8, 0xf5, + 0x5a, 0x5b, 0x73, 0x70, 0xde, 0x61, 0x7c, 0xb6, 0xce, 0x6f, 0x60, 0x3d, 0xa3, 0x81, 0xe4, 0xa2, + 0xae, 0xb6, 0xf6, 0x1c, 0x8c, 0x6b, 0x26, 0xbc, 0xa4, 0xec, 0x9d, 0x03, 0x54, 0xeb, 0x64, 0x15, + 0x16, 0x78, 0xaa, 0x4f, 0x8e, 0xed, 0x2e, 0xf0, 0x54, 0x4d, 0x9e, 0x23, 0x75, 0xd9, 0xe1, 0x71, + 0xb5, 0x5d, 0x63, 0xa9, 0x53, 0x1c, 0xfb, 0x6f, 0x78, 0x31, 0x7a, 0xa2, 0xa1, 0xbd, 0x2c, 0xe1, + 0xc2, 0x9c, 0x58, 0x34, 0x94, 0xf7, 0xdc, 0x8f, 0x72, 0x5a, 0x4c, 0x5a, 0xda, 0xe8, 0xfd, 0xa5, + 0x01, 0xed, 0xe2, 0x03, 0x89, 0x7c, 0x5e, 0x1f, 0xde, 0x9b, 0xef, 0xfe, 0x1e, 0x53, 0x41, 0xb8, + 0x99, 0x72, 0xc2, 0x7f, 0x5e, 0x4d, 0xf8, 0xff, 0x77, 0xb0, 0xf9, 0x0c, 0xa0, 0x60, 0x97, 0xbe, + 0xda, 0x6e, 0x1b, 0x33, 0xbb, 0xed, 0x42, 0x67, 0x1c, 0xf8, 0xde, 0xd8, 0x4f, 0x46, 0x11, 0xc5, + 0xb9, 0x74, 0xc5, 0x85, 0x71, 0xe0, 0xbf, 0x44, 0x4f, 0x01, 0xe0, 0xc3, 0x37, 0x34, 0x90, 0x99, + 0x6e, 0x0a, 0x02, 0x8e, 0xd1, 0xd3, 0xfb, 0xdb, 0x02, 0x74, 0x6a, 0xdf, 0x74, 0x6a, 0x72, 0x4f, + 0xfc, 0xb8, 0xc8, 0xa3, 0x7f, 0xab, 0xcb, 0x47, 0x4c, 0xf0, 0x2e, 0x31, 0x17, 0x53, 0x4b, 0x4c, + 0xf4, 0xa5, 0x40, 0x3e, 0x00, 0x10, 0x13, 0x2f, 0xf5, 0x83, 0x33, 0x6a, 0xe8, 0x2d, 0xd7, 0x16, + 0x93, 0x3e, 0x3a, 0xd4, 0x9d, 0x26, 0x26, 0x1e, 0x15, 0x82, 0x8b, 0xcc, 0xf4, 0xbe, 0x2d, 0x26, + 0x5f, 0x6a, 0xdb, 0xc4, 0x8e, 0x04, 0x57, 0x13, 0x88, 0x79, 0x06, 0xb6, 0x98, 0xbc, 0x40, 0x87, + 0xca, 0x2a, 0x8b, 0xac, 0x38, 0xf0, 0xb6, 0x64, 0x95, 0x55, 0x56, 0x59, 0x71, 0xe0, 0xb5, 0x65, + 0x3d, 0xab, 0x2c, 0xb3, 0xe2, 0xcc, 0xdb, 0x96, 0xb5, 0xac, 0xb2, 0xca, 0x6a, 0x17, 0xb1, 0x26, + 0x6b, 0xef, 0xef, 0x0d, 0xe8, 0xd4, 0xbe, 0x4e, 0x55, 0x03, 0x13, 0xe1, 0x65, 0x11, 0xa5, 0xa9, + 0xfa, 0x90, 0xc2, 0xab, 0x1b, 0x12, 0x31, 0x30, 0x1e, 0xc5, 0x97, 0x08, 0x4f, 0xe4, 0x49, 0x52, + 0x7c, 0x68, 0x59, 0xae, 0x9d, 0x08, 0x17, 0x1d, 0x66, 0x39, 0x93, 0x98, 0xae, 0x59, 0x2c, 0x0f, + 0xd0, 0x41, 0x7e, 0x09, 0x24, 0x11, 0x5e, 0x9e, 0xb0, 0x44, 0x52, 0x21, 0xf2, 0x54, 0xb2, 0x61, + 0xf9, 0x51, 0xb0, 0x9e, 0x88, 0xd3, 0xd9, 0x05, 0xf2, 0x48, 0xb3, 0x99, 0xcb, 0xc6, 0xb4, 0xac, + 0x9d, 0x88, 0x57, 0xfa, 0xe6, 0x38, 0x70, 0x7e, 0xf8, 0x71, 0xeb, 0xbd, 0x7f, 0xff, 0xb8, 0xf5, + 0xde, 0x9f, 0xa7, 0x5b, 0x8d, 0x1f, 0xa6, 0x5b, 0x8d, 0x7f, 0x4c, 0xb7, 0x1a, 0xff, 0x9d, 0x6e, + 0x35, 0x86, 0x4b, 0xfa, 0xcf, 0x95, 0x5f, 0xfd, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xc4, 0x4e, 0x24, + 0x22, 0xc4, 0x11, 0x00, 0x00, } func (m *Metrics) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -842,114 +848,138 @@ func (m *Metrics) Marshal() (dAtA []byte, err error) { } func (m *Metrics) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Metrics) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Hugetlb) > 0 { - for _, msg := range m.Hugetlb { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.MemoryOomControl != nil { + { + size, err := m.MemoryOomControl.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } - i += n + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } + i-- + dAtA[i] = 0x4a } - if m.Pids != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pids.Size())) - n1, err := m.Pids.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.CgroupStats != nil { + { + size, err := m.CgroupStats.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n1 + i-- + dAtA[i] = 0x42 } - if m.CPU != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.CPU.Size())) - n2, err := m.CPU.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if len(m.Network) > 0 { + for iNdEx := len(m.Network) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Network[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a } - i += n2 } - if m.Memory != nil { - dAtA[i] = 0x22 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Memory.Size())) - n3, err := m.Memory.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.Rdma != nil { + { + size, err := m.Rdma.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n3 + i-- + dAtA[i] = 0x32 } if m.Blkio != nil { - dAtA[i] = 0x2a - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Blkio.Size())) - n4, err := m.Blkio.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + { + size, err := m.Blkio.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n4 + i-- + dAtA[i] = 0x2a } - if m.Rdma != nil { - dAtA[i] = 0x32 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Rdma.Size())) - n5, err := m.Rdma.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.Memory != nil { + { + size, err := m.Memory.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n5 + i-- + dAtA[i] = 0x22 } - if len(m.Network) > 0 { - for _, msg := range m.Network { - dAtA[i] = 0x3a - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) + if m.CPU != nil { + { + size, err := m.CPU.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } - i += n + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } + i-- + dAtA[i] = 0x1a } - if m.CgroupStats != nil { - dAtA[i] = 0x42 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.CgroupStats.Size())) - n6, err := m.CgroupStats.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.Pids != nil { + { + size, err := m.Pids.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n6 + i-- + dAtA[i] = 0x12 } - if m.MemoryOomControl != nil { - dAtA[i] = 0x4a - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.MemoryOomControl.Size())) - n7, err := m.MemoryOomControl.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if len(m.Hugetlb) > 0 { + for iNdEx := len(m.Hugetlb) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Hugetlb[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa } - i += n7 - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) } - return i, nil + return len(dAtA) - i, nil } func (m *HugetlbStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -957,41 +987,48 @@ func (m *HugetlbStat) Marshal() (dAtA []byte, err error) { } func (m *HugetlbStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *HugetlbStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Usage != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Usage)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.Max != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) + if len(m.Pagesize) > 0 { + i -= len(m.Pagesize) + copy(dAtA[i:], m.Pagesize) + i = encodeVarintMetrics(dAtA, i, uint64(len(m.Pagesize))) + i-- + dAtA[i] = 0x22 } if m.Failcnt != 0 { - dAtA[i] = 0x18 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.Failcnt)) + i-- + dAtA[i] = 0x18 } - if len(m.Pagesize) > 0 { - dAtA[i] = 0x22 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(len(m.Pagesize))) - i += copy(dAtA[i:], m.Pagesize) + if m.Max != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) + i-- + dAtA[i] = 0x10 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Usage != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Usage)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *PidsStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -999,30 +1036,36 @@ func (m *PidsStat) Marshal() (dAtA []byte, err error) { } func (m *PidsStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PidsStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Current != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Current)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Limit != 0 { - dAtA[i] = 0x10 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x10 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Current != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Current)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *CPUStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1030,40 +1073,50 @@ func (m *CPUStat) Marshal() (dAtA []byte, err error) { } func (m *CPUStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CPUStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Usage != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Usage.Size())) - n7, err := m.Usage.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n7 + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Throttling != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Throttling.Size())) - n8, err := m.Throttling.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + { + size, err := m.Throttling.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n8 + i-- + dAtA[i] = 0x12 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Usage != nil { + { + size, err := m.Usage.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *CPUUsage) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1071,52 +1124,59 @@ func (m *CPUUsage) Marshal() (dAtA []byte, err error) { } func (m *CPUUsage) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CPUUsage) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Total != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Total)) - } - if m.Kernel != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Kernel)) - } - if m.User != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.User)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.PerCPU) > 0 { - dAtA10 := make([]byte, len(m.PerCPU)*10) - var j9 int + dAtA11 := make([]byte, len(m.PerCPU)*10) + var j10 int for _, num := range m.PerCPU { for num >= 1<<7 { - dAtA10[j9] = uint8(uint64(num)&0x7f | 0x80) + dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j9++ + j10++ } - dAtA10[j9] = uint8(num) - j9++ + dAtA11[j10] = uint8(num) + j10++ } + i -= j10 + copy(dAtA[i:], dAtA11[:j10]) + i = encodeVarintMetrics(dAtA, i, uint64(j10)) + i-- dAtA[i] = 0x22 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(j9)) - i += copy(dAtA[i:], dAtA10[:j9]) } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.User != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.User)) + i-- + dAtA[i] = 0x18 + } + if m.Kernel != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Kernel)) + i-- + dAtA[i] = 0x10 + } + if m.Total != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Total)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *Throttle) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1124,35 +1184,41 @@ func (m *Throttle) Marshal() (dAtA []byte, err error) { } func (m *Throttle) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Throttle) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Periods != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Periods)) - } - if m.ThrottledPeriods != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.ThrottledPeriods)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.ThrottledTime != 0 { - dAtA[i] = 0x18 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.ThrottledTime)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.ThrottledPeriods != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.ThrottledPeriods)) + i-- + dAtA[i] = 0x10 + } + if m.Periods != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Periods)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *MemoryStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1160,262 +1226,276 @@ func (m *MemoryStat) Marshal() (dAtA []byte, err error) { } func (m *MemoryStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *MemoryStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Cache != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Cache)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.RSS != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.RSS)) + if m.KernelTCP != nil { + { + size, err := m.KernelTCP.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0xa2 } - if m.RSSHuge != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.RSSHuge)) + if m.Kernel != nil { + { + size, err := m.Kernel.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x9a } - if m.MappedFile != 0 { - dAtA[i] = 0x20 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.MappedFile)) + if m.Swap != nil { + { + size, err := m.Swap.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x92 } - if m.Dirty != 0 { - dAtA[i] = 0x28 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Dirty)) + if m.Usage != nil { + { + size, err := m.Usage.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x8a } - if m.Writeback != 0 { - dAtA[i] = 0x30 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Writeback)) + if m.TotalUnevictable != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalUnevictable)) + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x80 } - if m.PgPgIn != 0 { - dAtA[i] = 0x38 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.PgPgIn)) + if m.TotalActiveFile != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalActiveFile)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xf8 } - if m.PgPgOut != 0 { - dAtA[i] = 0x40 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.PgPgOut)) + if m.TotalInactiveFile != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalInactiveFile)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xf0 } - if m.PgFault != 0 { - dAtA[i] = 0x48 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.PgFault)) + if m.TotalActiveAnon != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalActiveAnon)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xe8 } - if m.PgMajFault != 0 { - dAtA[i] = 0x50 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.PgMajFault)) - } - if m.InactiveAnon != 0 { - dAtA[i] = 0x58 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveAnon)) - } - if m.ActiveAnon != 0 { - dAtA[i] = 0x60 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveAnon)) - } - if m.InactiveFile != 0 { - dAtA[i] = 0x68 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveFile)) - } - if m.ActiveFile != 0 { - dAtA[i] = 0x70 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveFile)) - } - if m.Unevictable != 0 { - dAtA[i] = 0x78 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Unevictable)) - } - if m.HierarchicalMemoryLimit != 0 { - dAtA[i] = 0x80 - i++ + if m.TotalInactiveAnon != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalInactiveAnon)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.HierarchicalMemoryLimit)) + i-- + dAtA[i] = 0xe0 } - if m.HierarchicalSwapLimit != 0 { - dAtA[i] = 0x88 - i++ + if m.TotalPgMajFault != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgMajFault)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.HierarchicalSwapLimit)) + i-- + dAtA[i] = 0xd8 } - if m.TotalCache != 0 { - dAtA[i] = 0x90 - i++ + if m.TotalPgFault != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgFault)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalCache)) + i-- + dAtA[i] = 0xd0 } - if m.TotalRSS != 0 { - dAtA[i] = 0x98 - i++ + if m.TotalPgPgOut != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgPgOut)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalRSS)) + i-- + dAtA[i] = 0xc8 } - if m.TotalRSSHuge != 0 { - dAtA[i] = 0xa0 - i++ + if m.TotalPgPgIn != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgPgIn)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalRSSHuge)) + i-- + dAtA[i] = 0xc0 } - if m.TotalMappedFile != 0 { - dAtA[i] = 0xa8 - i++ + if m.TotalWriteback != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalWriteback)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalMappedFile)) + i-- + dAtA[i] = 0xb8 } if m.TotalDirty != 0 { - dAtA[i] = 0xb0 - i++ - dAtA[i] = 0x1 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.TotalDirty)) - } - if m.TotalWriteback != 0 { - dAtA[i] = 0xb8 - i++ + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalWriteback)) + i-- + dAtA[i] = 0xb0 } - if m.TotalPgPgIn != 0 { - dAtA[i] = 0xc0 - i++ + if m.TotalMappedFile != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalMappedFile)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgPgIn)) + i-- + dAtA[i] = 0xa8 } - if m.TotalPgPgOut != 0 { - dAtA[i] = 0xc8 - i++ + if m.TotalRSSHuge != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalRSSHuge)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgPgOut)) + i-- + dAtA[i] = 0xa0 } - if m.TotalPgFault != 0 { - dAtA[i] = 0xd0 - i++ + if m.TotalRSS != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalRSS)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgFault)) + i-- + dAtA[i] = 0x98 } - if m.TotalPgMajFault != 0 { - dAtA[i] = 0xd8 - i++ + if m.TotalCache != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TotalCache)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalPgMajFault)) + i-- + dAtA[i] = 0x90 } - if m.TotalInactiveAnon != 0 { - dAtA[i] = 0xe0 - i++ + if m.HierarchicalSwapLimit != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.HierarchicalSwapLimit)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalInactiveAnon)) + i-- + dAtA[i] = 0x88 } - if m.TotalActiveAnon != 0 { - dAtA[i] = 0xe8 - i++ + if m.HierarchicalMemoryLimit != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.HierarchicalMemoryLimit)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalActiveAnon)) + i-- + dAtA[i] = 0x80 } - if m.TotalInactiveFile != 0 { - dAtA[i] = 0xf0 - i++ - dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalInactiveFile)) + if m.Unevictable != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Unevictable)) + i-- + dAtA[i] = 0x78 } - if m.TotalActiveFile != 0 { - dAtA[i] = 0xf8 - i++ - dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalActiveFile)) + if m.ActiveFile != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveFile)) + i-- + dAtA[i] = 0x70 } - if m.TotalUnevictable != 0 { - dAtA[i] = 0x80 - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TotalUnevictable)) + if m.InactiveFile != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveFile)) + i-- + dAtA[i] = 0x68 } - if m.Usage != nil { - dAtA[i] = 0x8a - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Usage.Size())) - n11, err := m.Usage.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n11 + if m.ActiveAnon != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveAnon)) + i-- + dAtA[i] = 0x60 } - if m.Swap != nil { - dAtA[i] = 0x92 - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Swap.Size())) - n12, err := m.Swap.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n12 + if m.InactiveAnon != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveAnon)) + i-- + dAtA[i] = 0x58 } - if m.Kernel != nil { - dAtA[i] = 0x9a - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Kernel.Size())) - n13, err := m.Kernel.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n13 + if m.PgMajFault != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.PgMajFault)) + i-- + dAtA[i] = 0x50 } - if m.KernelTCP != nil { - dAtA[i] = 0xa2 - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.KernelTCP.Size())) - n14, err := m.KernelTCP.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n14 + if m.PgFault != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.PgFault)) + i-- + dAtA[i] = 0x48 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.PgPgOut != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.PgPgOut)) + i-- + dAtA[i] = 0x40 + } + if m.PgPgIn != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.PgPgIn)) + i-- + dAtA[i] = 0x38 + } + if m.Writeback != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Writeback)) + i-- + dAtA[i] = 0x30 + } + if m.Dirty != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Dirty)) + i-- + dAtA[i] = 0x28 + } + if m.MappedFile != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.MappedFile)) + i-- + dAtA[i] = 0x20 + } + if m.RSSHuge != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.RSSHuge)) + i-- + dAtA[i] = 0x18 + } + if m.RSS != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.RSS)) + i-- + dAtA[i] = 0x10 } - return i, nil + if m.Cache != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Cache)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil } func (m *MemoryEntry) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1423,40 +1503,46 @@ func (m *MemoryEntry) Marshal() (dAtA []byte, err error) { } func (m *MemoryEntry) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *MemoryEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Limit != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Limit)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.Usage != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Usage)) + if m.Failcnt != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Failcnt)) + i-- + dAtA[i] = 0x20 } if m.Max != 0 { - dAtA[i] = 0x18 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) + i-- + dAtA[i] = 0x18 } - if m.Failcnt != 0 { - dAtA[i] = 0x20 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Failcnt)) + if m.Usage != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Usage)) + i-- + dAtA[i] = 0x10 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Limit != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *MemoryOomControl) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1464,32 +1550,41 @@ func (m *MemoryOomControl) Marshal() (dAtA []byte, err error) { } func (m *MemoryOomControl) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *MemoryOomControl) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.OomKillDisable != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.OomKillDisable)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.OomKill != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.OomKill)) + i-- + dAtA[i] = 0x18 } if m.UnderOom != 0 { - dAtA[i] = 0x10 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.UnderOom)) + i-- + dAtA[i] = 0x10 } - if m.OomKill != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.OomKill)) + if m.OomKillDisable != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.OomKillDisable)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *BlkIOStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1497,116 +1592,138 @@ func (m *BlkIOStat) Marshal() (dAtA []byte, err error) { } func (m *BlkIOStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BlkIOStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.IoServiceBytesRecursive) > 0 { - for _, msg := range m.IoServiceBytesRecursive { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if len(m.IoServicedRecursive) > 0 { - for _, msg := range m.IoServicedRecursive { - dAtA[i] = 0x12 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if len(m.SectorsRecursive) > 0 { + for iNdEx := len(m.SectorsRecursive) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.SectorsRecursive[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x42 } } - if len(m.IoQueuedRecursive) > 0 { - for _, msg := range m.IoQueuedRecursive { - dAtA[i] = 0x1a - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if len(m.IoTimeRecursive) > 0 { + for iNdEx := len(m.IoTimeRecursive) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.IoTimeRecursive[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x3a } } - if len(m.IoServiceTimeRecursive) > 0 { - for _, msg := range m.IoServiceTimeRecursive { - dAtA[i] = 0x22 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if len(m.IoMergedRecursive) > 0 { + for iNdEx := len(m.IoMergedRecursive) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.IoMergedRecursive[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x32 } } if len(m.IoWaitTimeRecursive) > 0 { - for _, msg := range m.IoWaitTimeRecursive { - dAtA[i] = 0x2a - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.IoWaitTimeRecursive) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.IoWaitTimeRecursive[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x2a } } - if len(m.IoMergedRecursive) > 0 { - for _, msg := range m.IoMergedRecursive { - dAtA[i] = 0x32 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if len(m.IoServiceTimeRecursive) > 0 { + for iNdEx := len(m.IoServiceTimeRecursive) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.IoServiceTimeRecursive[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x22 } } - if len(m.IoTimeRecursive) > 0 { - for _, msg := range m.IoTimeRecursive { - dAtA[i] = 0x3a - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if len(m.IoQueuedRecursive) > 0 { + for iNdEx := len(m.IoQueuedRecursive) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.IoQueuedRecursive[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x1a } } - if len(m.SectorsRecursive) > 0 { - for _, msg := range m.SectorsRecursive { - dAtA[i] = 0x42 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if len(m.IoServicedRecursive) > 0 { + for iNdEx := len(m.IoServicedRecursive) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.IoServicedRecursive[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x12 } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.IoServiceBytesRecursive) > 0 { + for iNdEx := len(m.IoServiceBytesRecursive) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.IoServiceBytesRecursive[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } } - return i, nil + return len(dAtA) - i, nil } func (m *BlkIOEntry) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1614,47 +1731,55 @@ func (m *BlkIOEntry) Marshal() (dAtA []byte, err error) { } func (m *BlkIOEntry) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BlkIOEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Op) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(len(m.Op))) - i += copy(dAtA[i:], m.Op) - } - if len(m.Device) > 0 { - dAtA[i] = 0x12 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(len(m.Device))) - i += copy(dAtA[i:], m.Device) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.Major != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Major)) + if m.Value != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Value)) + i-- + dAtA[i] = 0x28 } if m.Minor != 0 { - dAtA[i] = 0x20 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.Minor)) + i-- + dAtA[i] = 0x20 } - if m.Value != 0 { - dAtA[i] = 0x28 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Value)) + if m.Major != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Major)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Device) > 0 { + i -= len(m.Device) + copy(dAtA[i:], m.Device) + i = encodeVarintMetrics(dAtA, i, uint64(len(m.Device))) + i-- + dAtA[i] = 0x12 } - return i, nil + if len(m.Op) > 0 { + i -= len(m.Op) + copy(dAtA[i:], m.Op) + i = encodeVarintMetrics(dAtA, i, uint64(len(m.Op))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *RdmaStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1662,44 +1787,54 @@ func (m *RdmaStat) Marshal() (dAtA []byte, err error) { } func (m *RdmaStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RdmaStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Current) > 0 { - for _, msg := range m.Current { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Limit) > 0 { - for _, msg := range m.Limit { - dAtA[i] = 0x12 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.Limit) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Limit[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x12 } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Current) > 0 { + for iNdEx := len(m.Current) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Current[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } } - return i, nil + return len(dAtA) - i, nil } func (m *RdmaEntry) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1707,36 +1842,43 @@ func (m *RdmaEntry) Marshal() (dAtA []byte, err error) { } func (m *RdmaEntry) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RdmaEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Device) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(len(m.Device))) - i += copy(dAtA[i:], m.Device) - } - if m.HcaHandles != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.HcaHandles)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.HcaObjects != 0 { - dAtA[i] = 0x18 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.HcaObjects)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.HcaHandles != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.HcaHandles)) + i-- + dAtA[i] = 0x10 + } + if len(m.Device) > 0 { + i -= len(m.Device) + copy(dAtA[i:], m.Device) + i = encodeVarintMetrics(dAtA, i, uint64(len(m.Device))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *NetworkStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1744,66 +1886,73 @@ func (m *NetworkStat) Marshal() (dAtA []byte, err error) { } func (m *NetworkStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NetworkStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Name) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(len(m.Name))) - i += copy(dAtA[i:], m.Name) - } - if m.RxBytes != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.RxBytes)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.RxPackets != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.RxPackets)) + if m.TxDropped != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TxDropped)) + i-- + dAtA[i] = 0x48 } - if m.RxErrors != 0 { - dAtA[i] = 0x20 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.RxErrors)) + if m.TxErrors != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TxErrors)) + i-- + dAtA[i] = 0x40 } - if m.RxDropped != 0 { - dAtA[i] = 0x28 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.RxDropped)) + if m.TxPackets != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.TxPackets)) + i-- + dAtA[i] = 0x38 } if m.TxBytes != 0 { - dAtA[i] = 0x30 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.TxBytes)) + i-- + dAtA[i] = 0x30 } - if m.TxPackets != 0 { - dAtA[i] = 0x38 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TxPackets)) + if m.RxDropped != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.RxDropped)) + i-- + dAtA[i] = 0x28 } - if m.TxErrors != 0 { - dAtA[i] = 0x40 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TxErrors)) + if m.RxErrors != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.RxErrors)) + i-- + dAtA[i] = 0x20 } - if m.TxDropped != 0 { - dAtA[i] = 0x48 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.TxDropped)) + if m.RxPackets != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.RxPackets)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.RxBytes != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.RxBytes)) + i-- + dAtA[i] = 0x10 + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintMetrics(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *CgroupStats) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1811,49 +1960,57 @@ func (m *CgroupStats) Marshal() (dAtA []byte, err error) { } func (m *CgroupStats) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CgroupStats) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.NrSleeping != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.NrSleeping)) - } - if m.NrRunning != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.NrRunning)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.NrStopped != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.NrStopped)) + if m.NrIoWait != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.NrIoWait)) + i-- + dAtA[i] = 0x28 } if m.NrUninterruptible != 0 { - dAtA[i] = 0x20 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.NrUninterruptible)) + i-- + dAtA[i] = 0x20 } - if m.NrIoWait != 0 { - dAtA[i] = 0x28 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.NrIoWait)) + if m.NrStopped != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.NrStopped)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.NrRunning != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.NrRunning)) + i-- + dAtA[i] = 0x10 } - return i, nil + if m.NrSleeping != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.NrSleeping)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil } func encodeVarintMetrics(dAtA []byte, offset int, v uint64) int { + offset -= sovMetrics(v) + base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) - return offset + 1 + return base } func (m *Metrics) Size() (n int) { if m == nil { @@ -2391,14 +2548,7 @@ func (m *CgroupStats) Size() (n int) { } func sovMetrics(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n + return (math_bits.Len64(x|1) + 6) / 7 } func sozMetrics(x uint64) (n int) { return sovMetrics(uint64((x << 1) ^ uint64((int64(x) >> 63)))) @@ -2407,16 +2557,26 @@ func (this *Metrics) String() string { if this == nil { return "nil" } + repeatedStringForHugetlb := "[]*HugetlbStat{" + for _, f := range this.Hugetlb { + repeatedStringForHugetlb += strings.Replace(f.String(), "HugetlbStat", "HugetlbStat", 1) + "," + } + repeatedStringForHugetlb += "}" + repeatedStringForNetwork := "[]*NetworkStat{" + for _, f := range this.Network { + repeatedStringForNetwork += strings.Replace(f.String(), "NetworkStat", "NetworkStat", 1) + "," + } + repeatedStringForNetwork += "}" s := strings.Join([]string{`&Metrics{`, - `Hugetlb:` + strings.Replace(fmt.Sprintf("%v", this.Hugetlb), "HugetlbStat", "HugetlbStat", 1) + `,`, - `Pids:` + strings.Replace(fmt.Sprintf("%v", this.Pids), "PidsStat", "PidsStat", 1) + `,`, - `CPU:` + strings.Replace(fmt.Sprintf("%v", this.CPU), "CPUStat", "CPUStat", 1) + `,`, - `Memory:` + strings.Replace(fmt.Sprintf("%v", this.Memory), "MemoryStat", "MemoryStat", 1) + `,`, - `MemoryOomControl:` + strings.Replace(fmt.Sprintf("%v", this.MemoryOomControl), "MemoryOomControl", "MemoryOomControl", 1) + `,`, - `Blkio:` + strings.Replace(fmt.Sprintf("%v", this.Blkio), "BlkIOStat", "BlkIOStat", 1) + `,`, - `Rdma:` + strings.Replace(fmt.Sprintf("%v", this.Rdma), "RdmaStat", "RdmaStat", 1) + `,`, - `Network:` + strings.Replace(fmt.Sprintf("%v", this.Network), "NetworkStat", "NetworkStat", 1) + `,`, - `CgroupStats:` + strings.Replace(fmt.Sprintf("%v", this.CgroupStats), "CgroupStats", "CgroupStats", 1) + `,`, + `Hugetlb:` + repeatedStringForHugetlb + `,`, + `Pids:` + strings.Replace(this.Pids.String(), "PidsStat", "PidsStat", 1) + `,`, + `CPU:` + strings.Replace(this.CPU.String(), "CPUStat", "CPUStat", 1) + `,`, + `Memory:` + strings.Replace(this.Memory.String(), "MemoryStat", "MemoryStat", 1) + `,`, + `Blkio:` + strings.Replace(this.Blkio.String(), "BlkIOStat", "BlkIOStat", 1) + `,`, + `Rdma:` + strings.Replace(this.Rdma.String(), "RdmaStat", "RdmaStat", 1) + `,`, + `Network:` + repeatedStringForNetwork + `,`, + `CgroupStats:` + strings.Replace(this.CgroupStats.String(), "CgroupStats", "CgroupStats", 1) + `,`, + `MemoryOomControl:` + strings.Replace(this.MemoryOomControl.String(), "MemoryOomControl", "MemoryOomControl", 1) + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -2453,8 +2613,8 @@ func (this *CPUStat) String() string { return "nil" } s := strings.Join([]string{`&CPUStat{`, - `Usage:` + strings.Replace(fmt.Sprintf("%v", this.Usage), "CPUUsage", "CPUUsage", 1) + `,`, - `Throttling:` + strings.Replace(fmt.Sprintf("%v", this.Throttling), "Throttle", "Throttle", 1) + `,`, + `Usage:` + strings.Replace(this.Usage.String(), "CPUUsage", "CPUUsage", 1) + `,`, + `Throttling:` + strings.Replace(this.Throttling.String(), "Throttle", "Throttle", 1) + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -2524,10 +2684,10 @@ func (this *MemoryStat) String() string { `TotalInactiveFile:` + fmt.Sprintf("%v", this.TotalInactiveFile) + `,`, `TotalActiveFile:` + fmt.Sprintf("%v", this.TotalActiveFile) + `,`, `TotalUnevictable:` + fmt.Sprintf("%v", this.TotalUnevictable) + `,`, - `Usage:` + strings.Replace(fmt.Sprintf("%v", this.Usage), "MemoryEntry", "MemoryEntry", 1) + `,`, - `Swap:` + strings.Replace(fmt.Sprintf("%v", this.Swap), "MemoryEntry", "MemoryEntry", 1) + `,`, - `Kernel:` + strings.Replace(fmt.Sprintf("%v", this.Kernel), "MemoryEntry", "MemoryEntry", 1) + `,`, - `KernelTCP:` + strings.Replace(fmt.Sprintf("%v", this.KernelTCP), "MemoryEntry", "MemoryEntry", 1) + `,`, + `Usage:` + strings.Replace(this.Usage.String(), "MemoryEntry", "MemoryEntry", 1) + `,`, + `Swap:` + strings.Replace(this.Swap.String(), "MemoryEntry", "MemoryEntry", 1) + `,`, + `Kernel:` + strings.Replace(this.Kernel.String(), "MemoryEntry", "MemoryEntry", 1) + `,`, + `KernelTCP:` + strings.Replace(this.KernelTCP.String(), "MemoryEntry", "MemoryEntry", 1) + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -2564,15 +2724,55 @@ func (this *BlkIOStat) String() string { if this == nil { return "nil" } + repeatedStringForIoServiceBytesRecursive := "[]*BlkIOEntry{" + for _, f := range this.IoServiceBytesRecursive { + repeatedStringForIoServiceBytesRecursive += strings.Replace(f.String(), "BlkIOEntry", "BlkIOEntry", 1) + "," + } + repeatedStringForIoServiceBytesRecursive += "}" + repeatedStringForIoServicedRecursive := "[]*BlkIOEntry{" + for _, f := range this.IoServicedRecursive { + repeatedStringForIoServicedRecursive += strings.Replace(f.String(), "BlkIOEntry", "BlkIOEntry", 1) + "," + } + repeatedStringForIoServicedRecursive += "}" + repeatedStringForIoQueuedRecursive := "[]*BlkIOEntry{" + for _, f := range this.IoQueuedRecursive { + repeatedStringForIoQueuedRecursive += strings.Replace(f.String(), "BlkIOEntry", "BlkIOEntry", 1) + "," + } + repeatedStringForIoQueuedRecursive += "}" + repeatedStringForIoServiceTimeRecursive := "[]*BlkIOEntry{" + for _, f := range this.IoServiceTimeRecursive { + repeatedStringForIoServiceTimeRecursive += strings.Replace(f.String(), "BlkIOEntry", "BlkIOEntry", 1) + "," + } + repeatedStringForIoServiceTimeRecursive += "}" + repeatedStringForIoWaitTimeRecursive := "[]*BlkIOEntry{" + for _, f := range this.IoWaitTimeRecursive { + repeatedStringForIoWaitTimeRecursive += strings.Replace(f.String(), "BlkIOEntry", "BlkIOEntry", 1) + "," + } + repeatedStringForIoWaitTimeRecursive += "}" + repeatedStringForIoMergedRecursive := "[]*BlkIOEntry{" + for _, f := range this.IoMergedRecursive { + repeatedStringForIoMergedRecursive += strings.Replace(f.String(), "BlkIOEntry", "BlkIOEntry", 1) + "," + } + repeatedStringForIoMergedRecursive += "}" + repeatedStringForIoTimeRecursive := "[]*BlkIOEntry{" + for _, f := range this.IoTimeRecursive { + repeatedStringForIoTimeRecursive += strings.Replace(f.String(), "BlkIOEntry", "BlkIOEntry", 1) + "," + } + repeatedStringForIoTimeRecursive += "}" + repeatedStringForSectorsRecursive := "[]*BlkIOEntry{" + for _, f := range this.SectorsRecursive { + repeatedStringForSectorsRecursive += strings.Replace(f.String(), "BlkIOEntry", "BlkIOEntry", 1) + "," + } + repeatedStringForSectorsRecursive += "}" s := strings.Join([]string{`&BlkIOStat{`, - `IoServiceBytesRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoServiceBytesRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, - `IoServicedRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoServicedRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, - `IoQueuedRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoQueuedRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, - `IoServiceTimeRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoServiceTimeRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, - `IoWaitTimeRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoWaitTimeRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, - `IoMergedRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoMergedRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, - `IoTimeRecursive:` + strings.Replace(fmt.Sprintf("%v", this.IoTimeRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, - `SectorsRecursive:` + strings.Replace(fmt.Sprintf("%v", this.SectorsRecursive), "BlkIOEntry", "BlkIOEntry", 1) + `,`, + `IoServiceBytesRecursive:` + repeatedStringForIoServiceBytesRecursive + `,`, + `IoServicedRecursive:` + repeatedStringForIoServicedRecursive + `,`, + `IoQueuedRecursive:` + repeatedStringForIoQueuedRecursive + `,`, + `IoServiceTimeRecursive:` + repeatedStringForIoServiceTimeRecursive + `,`, + `IoWaitTimeRecursive:` + repeatedStringForIoWaitTimeRecursive + `,`, + `IoMergedRecursive:` + repeatedStringForIoMergedRecursive + `,`, + `IoTimeRecursive:` + repeatedStringForIoTimeRecursive + `,`, + `SectorsRecursive:` + repeatedStringForSectorsRecursive + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -2597,9 +2797,19 @@ func (this *RdmaStat) String() string { if this == nil { return "nil" } + repeatedStringForCurrent := "[]*RdmaEntry{" + for _, f := range this.Current { + repeatedStringForCurrent += strings.Replace(f.String(), "RdmaEntry", "RdmaEntry", 1) + "," + } + repeatedStringForCurrent += "}" + repeatedStringForLimit := "[]*RdmaEntry{" + for _, f := range this.Limit { + repeatedStringForLimit += strings.Replace(f.String(), "RdmaEntry", "RdmaEntry", 1) + "," + } + repeatedStringForLimit += "}" s := strings.Join([]string{`&RdmaStat{`, - `Current:` + strings.Replace(fmt.Sprintf("%v", this.Current), "RdmaEntry", "RdmaEntry", 1) + `,`, - `Limit:` + strings.Replace(fmt.Sprintf("%v", this.Limit), "RdmaEntry", "RdmaEntry", 1) + `,`, + `Current:` + repeatedStringForCurrent + `,`, + `Limit:` + repeatedStringForLimit + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -3008,16 +3218,14 @@ func (m *Metrics) Unmarshal(dAtA []byte) error { if err := m.MemoryOomControl.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipMetrics(dAtA[iNdEx:]) if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3157,10 +3365,7 @@ func (m *HugetlbStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3249,10 +3454,7 @@ func (m *PidsStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3375,10 +3577,7 @@ func (m *CPUStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3562,10 +3761,7 @@ func (m *CPUUsage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3673,10 +3869,7 @@ func (m *Throttle) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -4479,10 +4672,7 @@ func (m *MemoryStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -4609,10 +4799,7 @@ func (m *MemoryEntry) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -4720,10 +4907,7 @@ func (m *MemoryOomControl) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -5046,10 +5230,7 @@ func (m *BlkIOStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -5221,10 +5402,7 @@ func (m *BlkIOEntry) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -5343,10 +5521,7 @@ func (m *RdmaStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -5467,10 +5642,7 @@ func (m *RdmaEntry) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -5705,10 +5877,7 @@ func (m *NetworkStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -5854,10 +6023,7 @@ func (m *CgroupStats) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -5876,6 +6042,7 @@ func (m *CgroupStats) Unmarshal(dAtA []byte) error { func skipMetrics(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 + depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -5907,10 +6074,8 @@ func skipMetrics(dAtA []byte) (n int, err error) { break } } - return iNdEx, nil case 1: iNdEx += 8 - return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -5931,55 +6096,30 @@ func skipMetrics(dAtA []byte) (n int, err error) { return 0, ErrInvalidLengthMetrics } iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthMetrics - } - return iNdEx, nil case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowMetrics - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipMetrics(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthMetrics - } - } - return iNdEx, nil + depth++ case 4: - return iNdEx, nil + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupMetrics + } + depth-- case 5: iNdEx += 4 - return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } + if iNdEx < 0 { + return 0, ErrInvalidLengthMetrics + } + if depth == 0 { + return iNdEx, nil + } } - panic("unreachable") + return 0, io.ErrUnexpectedEOF } var ( - ErrInvalidLengthMetrics = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowMetrics = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthMetrics = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowMetrics = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupMetrics = fmt.Errorf("proto: unexpected end of group") ) diff --git a/vendor/github.com/containerd/cgroups/subsystem.go b/vendor/github.com/containerd/cgroups/subsystem.go index 1349fc6675ba9..b2f41854d2c51 100644 --- a/vendor/github.com/containerd/cgroups/subsystem.go +++ b/vendor/github.com/containerd/cgroups/subsystem.go @@ -18,6 +18,7 @@ package cgroups import ( "fmt" + "os" v1 "github.com/containerd/cgroups/stats/v1" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -46,7 +47,6 @@ const ( // available on most linux systems func Subsystems() []Name { n := []Name{ - Hugetlb, Freezer, Pids, NetCLS, @@ -59,9 +59,12 @@ func Subsystems() []Name { Blkio, Rdma, } - if !isUserNS { + if !RunningInUserNS() { n = append(n, Devices) } + if _, err := os.Stat("/sys/kernel/mm/hugepages"); err == nil { + n = append(n, Hugetlb) + } return n } diff --git a/vendor/github.com/containerd/cgroups/utils.go b/vendor/github.com/containerd/cgroups/utils.go index fed9af02923b5..ed894b3ec46db 100644 --- a/vendor/github.com/containerd/cgroups/utils.go +++ b/vendor/github.com/containerd/cgroups/utils.go @@ -36,7 +36,8 @@ import ( ) var ( - isUserNS = runningInUserNS() + nsOnce sync.Once + inUserNS bool checkMode sync.Once cgMode CGMode ) @@ -81,33 +82,37 @@ func Mode() CGMode { return cgMode } -// runningInUserNS detects whether we are currently running in a user namespace. +// RunningInUserNS detects whether we are currently running in a user namespace. // Copied from github.com/lxc/lxd/shared/util.go -func runningInUserNS() bool { - file, err := os.Open("/proc/self/uid_map") - if err != nil { - // This kernel-provided file only exists if user namespaces are supported - return false - } - defer file.Close() +func RunningInUserNS() bool { + nsOnce.Do(func() { + file, err := os.Open("/proc/self/uid_map") + if err != nil { + // This kernel-provided file only exists if user namespaces are supported + return + } + defer file.Close() - buf := bufio.NewReader(file) - l, _, err := buf.ReadLine() - if err != nil { - return false - } + buf := bufio.NewReader(file) + l, _, err := buf.ReadLine() + if err != nil { + return + } - line := string(l) - var a, b, c int64 - fmt.Sscanf(line, "%d %d %d", &a, &b, &c) - /* - * We assume we are in the initial user namespace if we have a full - * range - 4294967295 uids starting at uid 0. - */ - if a == 0 && b == 0 && c == 4294967295 { - return false - } - return true + line := string(l) + var a, b, c int64 + fmt.Sscanf(line, "%d %d %d", &a, &b, &c) + + /* + * We assume we are in the initial user namespace if we have a full + * range - 4294967295 uids starting at uid 0. + */ + if a == 0 && b == 0 && c == 4294967295 { + return + } + inUserNS = true + }) + return inUserNS } // defaults returns all known groups @@ -132,7 +137,7 @@ func defaults(root string) ([]Subsystem, error) { } // only add the devices cgroup if we are not in a user namespace // because modifications are not allowed - if !isUserNS { + if !RunningInUserNS() { s = append(s, NewDevices(root)) } // add the hugetlb cgroup if error wasn't due to missing hugetlb diff --git a/vendor/github.com/containerd/cgroups/v2/ebpf.go b/vendor/github.com/containerd/cgroups/v2/ebpf.go index adda7550896ca..bd384818c527a 100644 --- a/vendor/github.com/containerd/cgroups/v2/ebpf.go +++ b/vendor/github.com/containerd/cgroups/v2/ebpf.go @@ -19,6 +19,7 @@ package v2 import ( "github.com/cilium/ebpf" "github.com/cilium/ebpf/asm" + "github.com/cilium/ebpf/link" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "golang.org/x/sys/unix" @@ -42,12 +43,23 @@ func LoadAttachCgroupDeviceFilter(insts asm.Instructions, license string, dirFD if err != nil { return nilCloser, err } - if err := prog.Attach(dirFD, ebpf.AttachCGroupDevice, unix.BPF_F_ALLOW_MULTI); err != nil { + err = link.RawAttachProgram(link.RawAttachProgramOptions{ + Target: dirFD, + Program: prog, + Attach: ebpf.AttachCGroupDevice, + Flags: unix.BPF_F_ALLOW_MULTI, + }) + if err != nil { return nilCloser, errors.Wrap(err, "failed to call BPF_PROG_ATTACH (BPF_CGROUP_DEVICE, BPF_F_ALLOW_MULTI)") } closer := func() error { - if err := prog.Detach(dirFD, ebpf.AttachCGroupDevice, unix.BPF_F_ALLOW_MULTI); err != nil { - return errors.Wrap(err, "failed to call BPF_PROG_DETACH (BPF_CGROUP_DEVICE, BPF_F_ALLOW_MULTI)") + err = link.RawDetachProgram(link.RawDetachProgramOptions{ + Target: dirFD, + Program: prog, + Attach: ebpf.AttachCGroupDevice, + }) + if err != nil { + return errors.Wrap(err, "failed to call BPF_PROG_DETACH (BPF_CGROUP_DEVICE)") } return nil } diff --git a/vendor/github.com/containerd/cgroups/v2/stats/metrics.pb.go b/vendor/github.com/containerd/cgroups/v2/stats/metrics.pb.go index dd54edb69a1f4..0bd493998f71f 100644 --- a/vendor/github.com/containerd/cgroups/v2/stats/metrics.pb.go +++ b/vendor/github.com/containerd/cgroups/v2/stats/metrics.pb.go @@ -9,6 +9,7 @@ import ( proto "github.com/gogo/protobuf/proto" io "io" math "math" + math_bits "math/bits" reflect "reflect" strings "strings" ) @@ -22,7 +23,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type Metrics struct { Pids *PidsStat `protobuf:"bytes,1,opt,name=pids,proto3" json:"pids,omitempty"` @@ -50,7 +51,7 @@ func (m *Metrics) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Metrics.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -90,7 +91,7 @@ func (m *PidsStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_PidsStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -134,7 +135,7 @@ func (m *CPUStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CPUStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -207,7 +208,7 @@ func (m *MemoryStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_MemoryStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -250,7 +251,7 @@ func (m *MemoryEvents) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_MemoryEvents.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -290,7 +291,7 @@ func (m *RdmaStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RdmaStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -331,7 +332,7 @@ func (m *RdmaEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RdmaEntry.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -370,7 +371,7 @@ func (m *IOStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_IOStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -414,7 +415,7 @@ func (m *IOEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_IOEntry.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -455,7 +456,7 @@ func (m *HugeTlbStat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return xxx_messageInfo_HugeTlbStat.Marshal(b, m, deterministic) } else { b = b[:cap(b)] - n, err := m.MarshalTo(b) + n, err := m.MarshalToSizedBuffer(b) if err != nil { return nil, err } @@ -573,7 +574,7 @@ var fileDescriptor_2fc6005842049e6b = []byte{ func (m *Metrics) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -581,92 +582,112 @@ func (m *Metrics) Marshal() (dAtA []byte, err error) { } func (m *Metrics) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Metrics) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Pids != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pids.Size())) - n1, err := m.Pids.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.MemoryEvents != nil { + { + size, err := m.MemoryEvents.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n1 + i-- + dAtA[i] = 0x42 } - if m.CPU != nil { - dAtA[i] = 0x12 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.CPU.Size())) - n2, err := m.CPU.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if len(m.Hugetlb) > 0 { + for iNdEx := len(m.Hugetlb) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Hugetlb[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a } - i += n2 } - if m.Memory != nil { - dAtA[i] = 0x22 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Memory.Size())) - n3, err := m.Memory.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.Io != nil { + { + size, err := m.Io.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n3 + i-- + dAtA[i] = 0x32 } if m.Rdma != nil { - dAtA[i] = 0x2a - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Rdma.Size())) - n4, err := m.Rdma.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + { + size, err := m.Rdma.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n4 + i-- + dAtA[i] = 0x2a } - if m.Io != nil { - dAtA[i] = 0x32 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Io.Size())) - n5, err := m.Io.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.Memory != nil { + { + size, err := m.Memory.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n5 + i-- + dAtA[i] = 0x22 } - if len(m.Hugetlb) > 0 { - for _, msg := range m.Hugetlb { - dAtA[i] = 0x3a - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) + if m.CPU != nil { + { + size, err := m.CPU.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } - i += n + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } + i-- + dAtA[i] = 0x12 } - if m.MemoryEvents != nil { - dAtA[i] = 0x42 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.MemoryEvents.Size())) - n6, err := m.MemoryEvents.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + if m.Pids != nil { + { + size, err := m.Pids.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n6 - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + i-- + dAtA[i] = 0xa } - return i, nil + return len(dAtA) - i, nil } func (m *PidsStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -674,30 +695,36 @@ func (m *PidsStat) Marshal() (dAtA []byte, err error) { } func (m *PidsStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PidsStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Current != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Current)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.Limit != 0 { - dAtA[i] = 0x10 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.Limit)) + i-- + dAtA[i] = 0x10 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Current != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Current)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *CPUStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -705,50 +732,56 @@ func (m *CPUStat) Marshal() (dAtA []byte, err error) { } func (m *CPUStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CPUStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.UsageUsec != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.UsageUsec)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.UserUsec != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.UserUsec)) + if m.ThrottledUsec != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.ThrottledUsec)) + i-- + dAtA[i] = 0x30 } - if m.SystemUsec != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.SystemUsec)) + if m.NrThrottled != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.NrThrottled)) + i-- + dAtA[i] = 0x28 } if m.NrPeriods != 0 { - dAtA[i] = 0x20 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.NrPeriods)) + i-- + dAtA[i] = 0x20 } - if m.NrThrottled != 0 { - dAtA[i] = 0x28 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.NrThrottled)) + if m.SystemUsec != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.SystemUsec)) + i-- + dAtA[i] = 0x18 } - if m.ThrottledUsec != 0 { - dAtA[i] = 0x30 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.ThrottledUsec)) + if m.UserUsec != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.UserUsec)) + i-- + dAtA[i] = 0x10 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.UsageUsec != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.UsageUsec)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *MemoryStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -756,235 +789,241 @@ func (m *MemoryStat) Marshal() (dAtA []byte, err error) { } func (m *MemoryStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *MemoryStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Anon != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Anon)) - } - if m.File != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.File)) - } - if m.KernelStack != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.KernelStack)) - } - if m.Slab != 0 { - dAtA[i] = 0x20 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Slab)) - } - if m.Sock != 0 { - dAtA[i] = 0x28 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Sock)) - } - if m.Shmem != 0 { - dAtA[i] = 0x30 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Shmem)) - } - if m.FileMapped != 0 { - dAtA[i] = 0x38 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.FileMapped)) - } - if m.FileDirty != 0 { - dAtA[i] = 0x40 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.FileDirty)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.FileWriteback != 0 { - dAtA[i] = 0x48 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.FileWriteback)) + if m.SwapLimit != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.SwapLimit)) + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x98 } - if m.AnonThp != 0 { - dAtA[i] = 0x50 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.AnonThp)) + if m.SwapUsage != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.SwapUsage)) + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x90 } - if m.InactiveAnon != 0 { - dAtA[i] = 0x58 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveAnon)) + if m.UsageLimit != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.UsageLimit)) + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x88 } - if m.ActiveAnon != 0 { - dAtA[i] = 0x60 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveAnon)) + if m.Usage != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Usage)) + i-- + dAtA[i] = 0x2 + i-- + dAtA[i] = 0x80 } - if m.InactiveFile != 0 { - dAtA[i] = 0x68 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveFile)) + if m.ThpCollapseAlloc != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.ThpCollapseAlloc)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xf8 } - if m.ActiveFile != 0 { - dAtA[i] = 0x70 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveFile)) + if m.ThpFaultAlloc != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.ThpFaultAlloc)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xf0 } - if m.Unevictable != 0 { - dAtA[i] = 0x78 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Unevictable)) + if m.Pglazyfreed != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Pglazyfreed)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xe8 } - if m.SlabReclaimable != 0 { - dAtA[i] = 0x80 - i++ + if m.Pglazyfree != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Pglazyfree)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.SlabReclaimable)) + i-- + dAtA[i] = 0xe0 } - if m.SlabUnreclaimable != 0 { - dAtA[i] = 0x88 - i++ + if m.Pgdeactivate != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Pgdeactivate)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.SlabUnreclaimable)) + i-- + dAtA[i] = 0xd8 } - if m.Pgfault != 0 { - dAtA[i] = 0x90 - i++ + if m.Pgactivate != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Pgactivate)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pgfault)) + i-- + dAtA[i] = 0xd0 } - if m.Pgmajfault != 0 { - dAtA[i] = 0x98 - i++ + if m.Pgsteal != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Pgsteal)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pgmajfault)) + i-- + dAtA[i] = 0xc8 } - if m.WorkingsetRefault != 0 { - dAtA[i] = 0xa0 - i++ + if m.Pgscan != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Pgscan)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.WorkingsetRefault)) + i-- + dAtA[i] = 0xc0 } - if m.WorkingsetActivate != 0 { - dAtA[i] = 0xa8 - i++ + if m.Pgrefill != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Pgrefill)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.WorkingsetActivate)) + i-- + dAtA[i] = 0xb8 } if m.WorkingsetNodereclaim != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.WorkingsetNodereclaim)) + i-- + dAtA[i] = 0x1 + i-- dAtA[i] = 0xb0 - i++ + } + if m.WorkingsetActivate != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.WorkingsetActivate)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.WorkingsetNodereclaim)) + i-- + dAtA[i] = 0xa8 } - if m.Pgrefill != 0 { - dAtA[i] = 0xb8 - i++ + if m.WorkingsetRefault != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.WorkingsetRefault)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pgrefill)) + i-- + dAtA[i] = 0xa0 } - if m.Pgscan != 0 { - dAtA[i] = 0xc0 - i++ + if m.Pgmajfault != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Pgmajfault)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pgscan)) + i-- + dAtA[i] = 0x98 } - if m.Pgsteal != 0 { - dAtA[i] = 0xc8 - i++ + if m.Pgfault != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Pgfault)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pgsteal)) + i-- + dAtA[i] = 0x90 } - if m.Pgactivate != 0 { - dAtA[i] = 0xd0 - i++ + if m.SlabUnreclaimable != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.SlabUnreclaimable)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pgactivate)) + i-- + dAtA[i] = 0x88 } - if m.Pgdeactivate != 0 { - dAtA[i] = 0xd8 - i++ + if m.SlabReclaimable != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.SlabReclaimable)) + i-- dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pgdeactivate)) + i-- + dAtA[i] = 0x80 + } + if m.Unevictable != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Unevictable)) + i-- + dAtA[i] = 0x78 + } + if m.ActiveFile != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveFile)) + i-- + dAtA[i] = 0x70 + } + if m.InactiveFile != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveFile)) + i-- + dAtA[i] = 0x68 + } + if m.ActiveAnon != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.ActiveAnon)) + i-- + dAtA[i] = 0x60 + } + if m.InactiveAnon != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.InactiveAnon)) + i-- + dAtA[i] = 0x58 + } + if m.AnonThp != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.AnonThp)) + i-- + dAtA[i] = 0x50 } - if m.Pglazyfree != 0 { - dAtA[i] = 0xe0 - i++ - dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pglazyfree)) + if m.FileWriteback != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.FileWriteback)) + i-- + dAtA[i] = 0x48 } - if m.Pglazyfreed != 0 { - dAtA[i] = 0xe8 - i++ - dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Pglazyfreed)) + if m.FileDirty != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.FileDirty)) + i-- + dAtA[i] = 0x40 } - if m.ThpFaultAlloc != 0 { - dAtA[i] = 0xf0 - i++ - dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.ThpFaultAlloc)) + if m.FileMapped != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.FileMapped)) + i-- + dAtA[i] = 0x38 } - if m.ThpCollapseAlloc != 0 { - dAtA[i] = 0xf8 - i++ - dAtA[i] = 0x1 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.ThpCollapseAlloc)) + if m.Shmem != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Shmem)) + i-- + dAtA[i] = 0x30 } - if m.Usage != 0 { - dAtA[i] = 0x80 - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Usage)) + if m.Sock != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Sock)) + i-- + dAtA[i] = 0x28 } - if m.UsageLimit != 0 { - dAtA[i] = 0x88 - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.UsageLimit)) + if m.Slab != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Slab)) + i-- + dAtA[i] = 0x20 } - if m.SwapUsage != 0 { - dAtA[i] = 0x90 - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.SwapUsage)) + if m.KernelStack != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.KernelStack)) + i-- + dAtA[i] = 0x18 } - if m.SwapLimit != 0 { - dAtA[i] = 0x98 - i++ - dAtA[i] = 0x2 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.SwapLimit)) + if m.File != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.File)) + i-- + dAtA[i] = 0x10 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Anon != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Anon)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *MemoryEvents) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -992,45 +1031,51 @@ func (m *MemoryEvents) Marshal() (dAtA []byte, err error) { } func (m *MemoryEvents) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *MemoryEvents) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Low != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Low)) - } - if m.High != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.High)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.Max != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) + if m.OomKill != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.OomKill)) + i-- + dAtA[i] = 0x28 } if m.Oom != 0 { - dAtA[i] = 0x20 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.Oom)) + i-- + dAtA[i] = 0x20 } - if m.OomKill != 0 { - dAtA[i] = 0x28 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.OomKill)) + if m.Max != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.High != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.High)) + i-- + dAtA[i] = 0x10 + } + if m.Low != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Low)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *RdmaStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1038,44 +1083,54 @@ func (m *RdmaStat) Marshal() (dAtA []byte, err error) { } func (m *RdmaStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RdmaStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Current) > 0 { - for _, msg := range m.Current { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err - } - i += n - } + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Limit) > 0 { - for _, msg := range m.Limit { - dAtA[i] = 0x12 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.Limit) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Limit[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0x12 } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if len(m.Current) > 0 { + for iNdEx := len(m.Current) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Current[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } } - return i, nil + return len(dAtA) - i, nil } func (m *RdmaEntry) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1083,36 +1138,43 @@ func (m *RdmaEntry) Marshal() (dAtA []byte, err error) { } func (m *RdmaEntry) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RdmaEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if len(m.Device) > 0 { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(len(m.Device))) - i += copy(dAtA[i:], m.Device) - } - if m.HcaHandles != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.HcaHandles)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if m.HcaObjects != 0 { - dAtA[i] = 0x18 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.HcaObjects)) + i-- + dAtA[i] = 0x18 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.HcaHandles != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.HcaHandles)) + i-- + dAtA[i] = 0x10 } - return i, nil + if len(m.Device) > 0 { + i -= len(m.Device) + copy(dAtA[i:], m.Device) + i = encodeVarintMetrics(dAtA, i, uint64(len(m.Device))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil } func (m *IOStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1120,32 +1182,40 @@ func (m *IOStat) Marshal() (dAtA []byte, err error) { } func (m *IOStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *IOStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } if len(m.Usage) > 0 { - for _, msg := range m.Usage { - dAtA[i] = 0xa - i++ - i = encodeVarintMetrics(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + for iNdEx := len(m.Usage) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Usage[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetrics(dAtA, i, uint64(size)) } - i += n + i-- + dAtA[i] = 0xa } } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + return len(dAtA) - i, nil } func (m *IOEntry) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1153,50 +1223,56 @@ func (m *IOEntry) Marshal() (dAtA []byte, err error) { } func (m *IOEntry) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *IOEntry) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Major != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Major)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } - if m.Minor != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Minor)) + if m.Wios != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Wios)) + i-- + dAtA[i] = 0x30 } - if m.Rbytes != 0 { - dAtA[i] = 0x18 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Rbytes)) + if m.Rios != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Rios)) + i-- + dAtA[i] = 0x28 } if m.Wbytes != 0 { - dAtA[i] = 0x20 - i++ i = encodeVarintMetrics(dAtA, i, uint64(m.Wbytes)) + i-- + dAtA[i] = 0x20 } - if m.Rios != 0 { - dAtA[i] = 0x28 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Rios)) + if m.Rbytes != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Rbytes)) + i-- + dAtA[i] = 0x18 } - if m.Wios != 0 { - dAtA[i] = 0x30 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Wios)) + if m.Minor != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Minor)) + i-- + dAtA[i] = 0x10 } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Major != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Major)) + i-- + dAtA[i] = 0x8 } - return i, nil + return len(dAtA) - i, nil } func (m *HugeTlbStat) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) if err != nil { return nil, err } @@ -1204,40 +1280,49 @@ func (m *HugeTlbStat) Marshal() (dAtA []byte, err error) { } func (m *HugeTlbStat) MarshalTo(dAtA []byte) (int, error) { - var i int + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *HugeTlbStat) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) _ = i var l int _ = l - if m.Current != 0 { - dAtA[i] = 0x8 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Current)) - } - if m.Max != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) } if len(m.Pagesize) > 0 { - dAtA[i] = 0x1a - i++ + i -= len(m.Pagesize) + copy(dAtA[i:], m.Pagesize) i = encodeVarintMetrics(dAtA, i, uint64(len(m.Pagesize))) - i += copy(dAtA[i:], m.Pagesize) + i-- + dAtA[i] = 0x1a } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) + if m.Max != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Max)) + i-- + dAtA[i] = 0x10 } - return i, nil + if m.Current != 0 { + i = encodeVarintMetrics(dAtA, i, uint64(m.Current)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil } func encodeVarintMetrics(dAtA []byte, offset int, v uint64) int { + offset -= sovMetrics(v) + base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) v >>= 7 offset++ } dAtA[offset] = uint8(v) - return offset + 1 + return base } func (m *Metrics) Size() (n int) { if m == nil { @@ -1590,14 +1675,7 @@ func (m *HugeTlbStat) Size() (n int) { } func sovMetrics(x uint64) (n int) { - for { - n++ - x >>= 7 - if x == 0 { - break - } - } - return n + return (math_bits.Len64(x|1) + 6) / 7 } func sozMetrics(x uint64) (n int) { return sovMetrics(uint64((x << 1) ^ uint64((int64(x) >> 63)))) @@ -1606,14 +1684,19 @@ func (this *Metrics) String() string { if this == nil { return "nil" } + repeatedStringForHugetlb := "[]*HugeTlbStat{" + for _, f := range this.Hugetlb { + repeatedStringForHugetlb += strings.Replace(f.String(), "HugeTlbStat", "HugeTlbStat", 1) + "," + } + repeatedStringForHugetlb += "}" s := strings.Join([]string{`&Metrics{`, - `Pids:` + strings.Replace(fmt.Sprintf("%v", this.Pids), "PidsStat", "PidsStat", 1) + `,`, - `CPU:` + strings.Replace(fmt.Sprintf("%v", this.CPU), "CPUStat", "CPUStat", 1) + `,`, - `Memory:` + strings.Replace(fmt.Sprintf("%v", this.Memory), "MemoryStat", "MemoryStat", 1) + `,`, - `Rdma:` + strings.Replace(fmt.Sprintf("%v", this.Rdma), "RdmaStat", "RdmaStat", 1) + `,`, - `Io:` + strings.Replace(fmt.Sprintf("%v", this.Io), "IOStat", "IOStat", 1) + `,`, - `Hugetlb:` + strings.Replace(fmt.Sprintf("%v", this.Hugetlb), "HugeTlbStat", "HugeTlbStat", 1) + `,`, - `MemoryEvents:` + strings.Replace(fmt.Sprintf("%v", this.MemoryEvents), "MemoryEvents", "MemoryEvents", 1) + `,`, + `Pids:` + strings.Replace(this.Pids.String(), "PidsStat", "PidsStat", 1) + `,`, + `CPU:` + strings.Replace(this.CPU.String(), "CPUStat", "CPUStat", 1) + `,`, + `Memory:` + strings.Replace(this.Memory.String(), "MemoryStat", "MemoryStat", 1) + `,`, + `Rdma:` + strings.Replace(this.Rdma.String(), "RdmaStat", "RdmaStat", 1) + `,`, + `Io:` + strings.Replace(this.Io.String(), "IOStat", "IOStat", 1) + `,`, + `Hugetlb:` + repeatedStringForHugetlb + `,`, + `MemoryEvents:` + strings.Replace(this.MemoryEvents.String(), "MemoryEvents", "MemoryEvents", 1) + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -1711,9 +1794,19 @@ func (this *RdmaStat) String() string { if this == nil { return "nil" } + repeatedStringForCurrent := "[]*RdmaEntry{" + for _, f := range this.Current { + repeatedStringForCurrent += strings.Replace(f.String(), "RdmaEntry", "RdmaEntry", 1) + "," + } + repeatedStringForCurrent += "}" + repeatedStringForLimit := "[]*RdmaEntry{" + for _, f := range this.Limit { + repeatedStringForLimit += strings.Replace(f.String(), "RdmaEntry", "RdmaEntry", 1) + "," + } + repeatedStringForLimit += "}" s := strings.Join([]string{`&RdmaStat{`, - `Current:` + strings.Replace(fmt.Sprintf("%v", this.Current), "RdmaEntry", "RdmaEntry", 1) + `,`, - `Limit:` + strings.Replace(fmt.Sprintf("%v", this.Limit), "RdmaEntry", "RdmaEntry", 1) + `,`, + `Current:` + repeatedStringForCurrent + `,`, + `Limit:` + repeatedStringForLimit + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -1736,8 +1829,13 @@ func (this *IOStat) String() string { if this == nil { return "nil" } + repeatedStringForUsage := "[]*IOEntry{" + for _, f := range this.Usage { + repeatedStringForUsage += strings.Replace(f.String(), "IOEntry", "IOEntry", 1) + "," + } + repeatedStringForUsage += "}" s := strings.Join([]string{`&IOStat{`, - `Usage:` + strings.Replace(fmt.Sprintf("%v", this.Usage), "IOEntry", "IOEntry", 1) + `,`, + `Usage:` + repeatedStringForUsage + `,`, `XXX_unrecognized:` + fmt.Sprintf("%v", this.XXX_unrecognized) + `,`, `}`, }, "") @@ -2065,10 +2163,7 @@ func (m *Metrics) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -2157,10 +2252,7 @@ func (m *PidsStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -2325,10 +2417,7 @@ func (m *CPUStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3044,10 +3133,7 @@ func (m *MemoryStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3193,10 +3279,7 @@ func (m *MemoryEvents) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3315,10 +3398,7 @@ func (m *RdmaStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3439,10 +3519,7 @@ func (m *RdmaEntry) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3527,10 +3604,7 @@ func (m *IOStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3695,10 +3769,7 @@ func (m *IOEntry) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3819,10 +3890,7 @@ func (m *HugeTlbStat) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthMetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthMetrics } if (iNdEx + skippy) > l { @@ -3841,6 +3909,7 @@ func (m *HugeTlbStat) Unmarshal(dAtA []byte) error { func skipMetrics(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 + depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -3872,10 +3941,8 @@ func skipMetrics(dAtA []byte) (n int, err error) { break } } - return iNdEx, nil case 1: iNdEx += 8 - return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -3896,55 +3963,30 @@ func skipMetrics(dAtA []byte) (n int, err error) { return 0, ErrInvalidLengthMetrics } iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthMetrics - } - return iNdEx, nil case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowMetrics - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipMetrics(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthMetrics - } - } - return iNdEx, nil + depth++ case 4: - return iNdEx, nil + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupMetrics + } + depth-- case 5: iNdEx += 4 - return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } + if iNdEx < 0 { + return 0, ErrInvalidLengthMetrics + } + if depth == 0 { + return iNdEx, nil + } } - panic("unreachable") + return 0, io.ErrUnexpectedEOF } var ( - ErrInvalidLengthMetrics = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowMetrics = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthMetrics = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowMetrics = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupMetrics = fmt.Errorf("proto: unexpected end of group") )