diff --git a/cmd/trace-runner/root.go b/cmd/trace-runner/root.go new file mode 100644 index 00000000..3444ef30 --- /dev/null +++ b/cmd/trace-runner/root.go @@ -0,0 +1,19 @@ +package main + +import ( + "os" + + "github.com/fntlnz/kubectl-trace/pkg/cmd" + "github.com/spf13/pflag" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" +) + +func main() { + flags := pflag.NewFlagSet("trace-runner", pflag.ExitOnError) + pflag.CommandLine = flags + + root := cmd.NewTraceRunnerCommand() + if err := root.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/go.mod b/go.mod index 6bf5f037..bfd708fd 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,14 @@ module github.com/fntlnz/kubectl-trace require ( cloud.google.com/go v0.34.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect + github.com/davecgh/go-spew v1.1.1 github.com/docker/distribution v2.6.2+incompatible // indirect github.com/docker/docker v0.7.3-0.20181124105010-0b7cb16dde4a // indirect github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 // indirect github.com/elazarl/goproxy v0.0.0-20181111060418-2ce16c963a8a // indirect github.com/evanphx/json-patch v4.1.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect + github.com/fntlnz/mountinfo v0.0.0-20171106231217-40cb42681fad github.com/ghodss/yaml v1.0.0 // indirect github.com/go-openapi/spec v0.17.2 // indirect github.com/gogo/protobuf v1.1.1 // indirect diff --git a/go.sum b/go.sum index 6eb69718..996cfd75 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/evanphx/json-patch v4.1.0+incompatible h1:K1MDoo4AZ4wU0GIU/fPmtZg7Vpz github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fntlnz/mountinfo v0.0.0-20171106231217-40cb42681fad h1:7dkG+DBBIETkv0nraI5oMvN4M5X3i75q7xq68eIq5Ag= +github.com/fntlnz/mountinfo v0.0.0-20171106231217-40cb42681fad/go.mod h1:OJmEqKcMeJq0teE8CysGMs/5Ulch9FogT/MmOzE1U9o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= diff --git a/pkg/cmd/tracerunner.go b/pkg/cmd/tracerunner.go new file mode 100644 index 00000000..9ed90179 --- /dev/null +++ b/pkg/cmd/tracerunner.go @@ -0,0 +1,208 @@ +package cmd + +import ( + "encoding/hex" + "fmt" + "io" + "math/rand" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "syscall" + + "github.com/fntlnz/mountinfo" + "github.com/spf13/cobra" + "golang.org/x/sys/unix" +) + +const runFolder = "/var/run" + +type TraceRunnerOptions struct { + podUID string + containerName string + inPod bool + programPath string + bpftraceBinaryPath string +} + +func NewTraceRunnerOptions() *TraceRunnerOptions { + return &TraceRunnerOptions{} +} + +func NewTraceRunnerCommand() *cobra.Command { + o := NewTraceRunnerOptions() + cmd := &cobra.Command{ + PreRunE: func(c *cobra.Command, args []string) error { + return o.Validate(c, args) + }, + RunE: func(c *cobra.Command, args []string) error { + if err := o.Complete(c, args); err != nil { + return err + } + if err := o.Run(); err != nil { + fmt.Fprintln(os.Stdout, err.Error()) + return nil + } + return nil + }, + } + + cmd.Flags().StringVarP(&o.containerName, "container", "c", o.containerName, "Specify the container") + cmd.Flags().StringVarP(&o.podUID, "poduid", "p", o.podUID, "Specify the pod UID") + cmd.Flags().StringVarP(&o.programPath, "program", "f", "program.bt", "Specify the bpftrace program path") + cmd.Flags().StringVarP(&o.bpftraceBinaryPath, "bpftracebinary", "b", "/bin/bpftrace", "Specify the bpftrace binary path") + cmd.Flags().BoolVar(&o.inPod, "inpod", false, "Wether or not run this bpftrace in a pod's container process namespace") + return cmd +} + +func (o *TraceRunnerOptions) Validate(cmd *cobra.Command, args []string) error { + // TODO(fntlnz): do some more meaningful validation here, for now just checking if they are there + if o.inPod == true && (len(o.containerName) == 0 || len(o.podUID) == 0) { + return fmt.Errorf("poduid and container must be specified when inpod=true") + } + return nil +} + +func (o *TraceRunnerOptions) Complete(cmd *cobra.Command, args []string) error { + return nil +} + +func (o *TraceRunnerOptions) Run() error { + if o.inPod == false { + c := exec.Command(o.bpftraceBinaryPath, o.programPath) + c.Stdout = os.Stdout + c.Stdin = os.Stdin + c.Stderr = os.Stderr + return c.Run() + } + + pid, err := findPidByPodContainer(o.podUID, o.containerName) + if err != nil { + return err + } + if pid == nil { + return fmt.Errorf("pid not found") + } + if len(*pid) == 0 { + return fmt.Errorf("invalid pid found") + } + + // pid found, enter its process namespace + pidns := path.Join("/proc", *pid, "/ns/pid") + pidnsfd, err := syscall.Open(pidns, syscall.O_RDONLY, 0666) + if err != nil { + return fmt.Errorf("error retrieving process namespace %s %v", pidns, err) + } + defer syscall.Close(pidnsfd) + syscall.RawSyscall(unix.SYS_SETNS, uintptr(pidnsfd), 0, 0) + + rootfs := path.Join("/proc", *pid, "root") + bpftracebinaryName, err := temporaryFileName("bpftrace") + if err != nil { + return err + } + temporaryProgramName := fmt.Sprintf("%s-%s", bpftracebinaryName, "program.bt") + + binaryPathProcRootfs := path.Join(rootfs, bpftracebinaryName) + if err := copyFile(o.bpftraceBinaryPath, binaryPathProcRootfs, 0755); err != nil { + return err + } + + programPathProcRootfs := path.Join(rootfs, temporaryProgramName) + if err := copyFile(o.programPath, programPathProcRootfs, 0644); err != nil { + return err + } + + if err := syscall.Chroot(rootfs); err != nil { + os.Remove(binaryPathProcRootfs) + return err + } + + defer os.Remove(bpftracebinaryName) + + c := exec.Command(bpftracebinaryName, temporaryProgramName) + + c.Stdout = os.Stdout + c.Stdin = os.Stdin + c.Stderr = os.Stderr + + return c.Run() +} + +func copyFile(src, dest string, mode os.FileMode) error { + in, err := os.Open(src) + if err != nil { + return fmt.Errorf("bpftrace binary not found in host: %v", err) + } + defer in.Close() + + out, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) + + if err != nil { + return fmt.Errorf("unable to create file in destination: %v", err) + } + defer out.Close() + + if _, err = io.Copy(out, in); err != nil { + return fmt.Errorf("unable to copy file to destination: %v", err) + } + + err = out.Sync() + + if err != nil { + return err + } + return nil +} + +func findPidByPodContainer(podUID, containerName string) (*string, error) { + d, err := os.Open("/proc") + + if err != nil { + return nil, err + } + + defer d.Close() + + for { + dirs, err := d.Readdir(10) + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + for _, di := range dirs { + if !di.IsDir() { + continue + } + dname := di.Name() + if dname[0] < '0' || dname[0] > '9' { + continue + } + + mi, err := mountinfo.GetMountInfo(path.Join("/proc", dname, "mountinfo")) + if err != nil { + continue + } + + for _, m := range mi { + root := m.Root + if strings.Contains(root, podUID) && strings.Contains(root, containerName) { + return &dname, nil + } + } + } + } + + return nil, fmt.Errorf("no process found for specified pod and container") +} + +func temporaryFileName(prefix string) (string, error) { + randBytes := make([]byte, 16) + rand.Read(randBytes) + return filepath.Join(runFolder, prefix+hex.EncodeToString(randBytes)), nil +} diff --git a/pkg/tracejob/job.go b/pkg/tracejob/job.go index 33e981fe..b35825c6 100644 --- a/pkg/tracejob/job.go +++ b/pkg/tracejob/job.go @@ -170,13 +170,10 @@ func (t *TraceJobClient) DeleteJobs(nf TraceJobFilter) error { return nil } -// todo(fntlnz): deal with programs that needs the user to send a signal to complete, -// like how the hist() function does -// Will likely need to allocate a TTY for this one thing. func (t *TraceJobClient) CreateJob(nj TraceJob) (*batchv1.Job, error) { bpfTraceCmd := []string{ - "bpftrace", - "/programs/program.bt", + "/bin/trace-runner", + "--program=/programs/program.bt", } commonMeta := metav1.ObjectMeta{ @@ -205,11 +202,7 @@ func (t *TraceJobClient) CreateJob(nj TraceJob) (*batchv1.Job, error) { TTLSecondsAfterFinished: int32Ptr(5), Parallelism: int32Ptr(1), Completions: int32Ptr(1), - // This is why your tracing job is being killed after 100 seconds, - // someone should work on it to make it configurable and let it run - // indefinitely by default. - ActiveDeadlineSeconds: int64Ptr(100), // TODO(fntlnz): allow canceling from kubectl and increase this, - BackoffLimit: int32Ptr(1), + BackoffLimit: int32Ptr(1), Template: apiv1.PodTemplateSpec{ ObjectMeta: commonMeta, Spec: apiv1.PodSpec{ diff --git a/program.bt b/program.bt new file mode 100644 index 00000000..b6d8bdd1 --- /dev/null +++ b/program.bt @@ -0,0 +1 @@ +uretprobe:/caturday:"main.counterValue" { printf("%d\n", retval) }'