Add retraced commands [ch3179]

Makefile

@@ -17,12 +17,25 @@ test:
 	go test -v ./pkg/...
 
 integration-test:
-	ginkgo -v -r -p --skip="docker container" tests/ginkgo
+	ginkgo -v -r -p --skip="docker container|retraced.events" tests/ginkgo
 
 integration-test-docker:
 	docker pull ubuntu:latest
 	ginkgo -v -r -p --focus="docker container" tests/ginkgo
 
+# this task assumes a working retraced installation, and requires the following params to be set:
+#
+#  RETRACED_API_ENDPOINT
+#  RETRACED_PROJECT_ID
+#  RETRACED_API_KEY
+#
+# Can also optionally set
+#
+#  RETRACED_INSECURE_SKIP_VERIFY=1
+#
+integration-test-retraced:
+	ginkgo -v -r -p --focus="retraced.events" tests/ginkgo
+
 build:
 	mkdir -p bin
 	go build -o ./bin/support-bundle .

cmd/support-bundle.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"os"
 
-	homedir "github.com/mitchellh/go-homedir"
+	"github.com/mitchellh/go-homedir"
 	"github.com/replicatedcom/support-bundle/pkg/cli"
 	"github.com/replicatedcom/support-bundle/pkg/cli/commands"
 	"github.com/spf13/cobra"

pkg/cli/generate.go

@@ -11,6 +11,7 @@ import (
 	"github.com/replicatedcom/support-bundle/pkg/spec"
 	"github.com/replicatedcom/support-bundle/pkg/types"
 	jww "github.com/spf13/jwalterweatherman"
+	"github.com/replicatedcom/support-bundle/pkg/plugins/retraced"
 )
 
 func (cli *Cli) Generate(cfgFiles []string, cfgDocs []string, bundlePath string, skipDefault bool, timeoutSeconds int) error {
@@ -52,10 +53,12 @@ func (cli *Cli) Generate(cfgFiles []string, cfgDocs []string, bundlePath string,
 	if err != nil {
 		return errors.Wrap(err, "Failed to initialize docker plugin")
 	}
+
 	planner := bundle.Planner{
 		Plugins: map[string]types.Plugin{
-			"core":   core.New(),
-			"docker": d,
+			"core":     core.New(),
+			"docker":   d,
+			"retraced": retraced.New(),
 		},
 	}
 

pkg/plugins/retraced/planners/events.go

@@ -0,0 +1,31 @@
+package planners
+
+import (
+	"github.com/pkg/errors"
+	"github.com/replicatedcom/support-bundle/pkg/plans"
+	"github.com/replicatedcom/support-bundle/pkg/plugins/retraced/producers"
+	"github.com/replicatedcom/support-bundle/pkg/types"
+)
+
+func Events(spec types.Spec) []types.Task {
+	var err error
+	if spec.RetracedEventsCommand == nil {
+		err = errors.New("spec for retraced.events required")
+	} else if spec.RetracedEventsCommand.APIEndpoint == "" { // require an endpoint, I don't think defaulting to SaaS Retraced makes sense
+		err = errors.New("retraced.events spec missing api_endpoint")
+	} else if spec.RetracedEventsCommand.APIToken == "" {
+		err = errors.New("retraced.events spec missing api_token")
+	} else if spec.RetracedEventsCommand.ProjectID == "" {
+		err = errors.New("retraced.events spec missing project_id")
+	}
+	if err != nil {
+		task := plans.PreparedError(err, spec)
+		return []types.Task{task}
+	}
+	return []types.Task{
+		&plans.StreamSource{
+			Producer: producers.Events(spec),
+			RawPath:  spec.Raw,
+		},
+	}
+}

pkg/plugins/retraced/plugin.go

@@ -0,0 +1,12 @@
+package retraced
+
+import (
+	"github.com/replicatedcom/support-bundle/pkg/plugins/retraced/planners"
+	"github.com/replicatedcom/support-bundle/pkg/types"
+)
+
+func New() types.Plugin {
+	return map[string]types.Planner{
+		"events": planners.Events,
+	}
+}

pkg/plugins/retraced/producers/events.go

@@ -0,0 +1,88 @@
+package producers
+
+import (
+	"context"
+	"crypto/tls"
+	"io"
+	"net/http"
+
+	"github.com/pkg/errors"
+	"github.com/replicatedcom/support-bundle/pkg/types"
+	"github.com/retracedhq/retraced-go"
+	"github.com/spf13/jwalterweatherman"
+)
+
+var (
+	defaultMask = &retraced.EventNodeMask{
+		Action:        true,
+		CRUD:          true,
+		CanonicalTime: true,
+	}
+	defaultQuery = &retraced.StructuredQuery{
+		CRUD: "c,u,d",
+	}
+)
+
+func Events(spec types.Spec) types.StreamProducer {
+	return func(ctx context.Context) (io.Reader, error) {
+
+		client, err := getClient(spec)
+		if err != nil {
+			return nil, errors.Wrap(err, "create retraced client")
+		}
+
+		query := getQuery(spec)
+		mask := getMask(spec)
+		reader, writer := io.Pipe()
+
+		go func() {
+			var err error
+			defer func() {
+				writer.CloseWithError(err)
+			}()
+			err = client.ExportCSV(ctx, writer, query, mask)
+			if err != nil {
+				jwalterweatherman.ERROR.Printf("Failed to collect retraced events %v", err)
+			}
+		}()
+
+		return reader, nil
+	}
+}
+func getClient(spec types.Spec) (*retraced.Client, error) {
+	client, err := retraced.NewClient(spec.RetracedEventsCommand.ProjectID, spec.RetracedEventsCommand.APIToken)
+	if err != nil {
+		return nil, err
+	}
+
+	if spec.RetracedEventsCommand.Insecure {
+		client.HttpClient = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					InsecureSkipVerify: true,
+				},
+			},
+		}
+	}
+	return client, nil
+}
+
+func getQuery(spec types.Spec) *retraced.StructuredQuery {
+	var query *retraced.StructuredQuery
+	if spec.RetracedEventsCommand.Query != nil {
+		query = spec.RetracedEventsCommand.Query
+	} else {
+		query = defaultQuery
+	}
+	return query
+}
+
+func getMask(spec types.Spec) *retraced.EventNodeMask {
+	var mask *retraced.EventNodeMask
+	if spec.RetracedEventsCommand.Mask != nil {
+		mask = spec.RetracedEventsCommand.Mask
+	} else {
+		mask = defaultMask
+	}
+	return mask
+}

pkg/types/spec.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	dockertypes "github.com/docker/docker/api/types"
+	"github.com/retracedhq/retraced-go"
 )
 
 type Doc struct {
@@ -26,6 +27,7 @@ type Spec struct {
 	DockerContainerLogs    *DockerContainerLogsOptions    `json:"docker.container-logs,omitempty"`
 	DockerContainerInspect *DockerContainerInspectOptions `json:"docker.container-inspect,omitempty"`
 	HTTPRequestCommand     *HTTPRequestCommandOptions     `json:"core.http-request,omitempty"`
+	RetracedEventsCommand  *RetracedEventsOptions         `json:"retraced.events,omitempty"`
 }
 
 type Config struct {
@@ -60,6 +62,15 @@ type HTTPRequestCommandOptions struct {
 	Insecure bool                `json:"insecure,omitempty"`
 }
 
+type RetracedEventsOptions struct {
+	APIEndpoint string                    `json:"api_endpoint"`
+	ProjectID   string                    `json:"project_id,omitempty"`
+	APIToken    string                    `json:"api_token,omitempty"`
+	Insecure    bool                      `json:"insecure,omitempty"`
+	Mask        *retraced.EventNodeMask   `json:"mask,omitempty"`
+	Query       *retraced.StructuredQuery `json:"query,omitempty"`
+}
+
 type Scrub struct {
 	Regex   string `json:"regex"`
 	Replace string `json:"replace"`

tests/ginkgo/retraced_audit_test.go

@@ -0,0 +1,88 @@
+package ginkgo
+
+import (
+	"fmt"
+	"os"
+
+	"strings"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/replicatedcom/support-bundle/pkg/types"
+)
+
+var _ = Describe("The retraced.events spec", func() {
+	opts := types.RetracedEventsOptions{
+		APIEndpoint: os.Getenv("RETRACED_API_ENDPOINT"),
+		APIToken:    os.Getenv("RETRACED_API_TOKEN"),
+		ProjectID:   os.Getenv("RETRACED_PROJECT_ID"),
+		Insecure:    os.Getenv("RETRACED_INSECURE_SKIP_VERIFY") != "",
+	}
+
+	BeforeEach(EnterNewTempDir)
+	BeforeEach(func() {
+		Expect(opts.APIEndpoint).NotTo(BeEmpty(), "RETRACED_API_ENDPOINT must be set")
+		Expect(opts.APIToken).NotTo(BeEmpty(), "RETRACED_API_TOKEN must be set")
+		Expect(opts.ProjectID).NotTo(BeEmpty(), "RETRACED_PROJECT_ID must be set")
+	})
+	AfterEach(CleanupDir)
+
+	It("Collects events from retraced", func() {
+		WriteBundleConfig(`
+specs:
+  - builtin: retraced.events
+    raw: /audit/events.txt
+    retraced.events:
+        api_endpoint: ` + opts.APIEndpoint + `
+        api_token: ` + opts.APIToken + `
+        project_id: ` + opts.ProjectID + `
+        insecure: ` + fmt.Sprintf("%v", opts.Insecure))
+
+		GenerateBundle()
+
+		Expect(err).NotTo(HaveOccurred())
+		errors := GetFileFromBundle("error.json")
+		Expect(errors).To(Equal("null"))
+
+		contents := GetFileFromBundle("audit/events.txt")
+		Expect(contents).ToNot(BeEmpty())
+
+		header := strings.Split(contents, "\n")[0]
+		Expect(header).To(Equal("action,crud,canonical_time"))
+
+	})
+
+	It("Allows for a custom mask+query", func() {
+		WriteBundleConfig(`
+specs:
+  - builtin: retraced.events
+    raw: /audit/events.txt
+    retraced.events:
+        api_endpoint: ` + opts.APIEndpoint + `
+        api_token: ` + opts.APIToken + `
+        project_id: ` + opts.ProjectID + `
+        insecure: ` + fmt.Sprintf("%v", opts.Insecure) + `
+        mask:
+          CRUD: true
+        query:
+          CRUD: r`)
+
+		GenerateBundle()
+
+		Expect(err).NotTo(HaveOccurred())
+		errors := GetFileFromBundle("error.json")
+		Expect(errors).To(Equal("null"))
+
+		contents := GetFileFromBundle("audit/events.txt")
+		Expect(contents).ToNot(BeEmpty())
+
+		lines := strings.Split(contents, "\n")
+		Expect(lines[0]).To(Equal("crud"))
+		for _, line := range lines[1:] {
+			if line != "" {
+				Expect(line).To(Equal("r"))
+			}
+		}
+
+	})
+})