Webhook message formatting using go templates (#582)

* feat: Refactor sendToWebhook().

Signed-off-by: Simarpreet Singh <simar@linux.com>

* feat: Add a simple barebones template.

Signed-off-by: Simarpreet Singh <simar@linux.com>

* feat: Add some sad paths for template output

Signed-off-by: Simarpreet Singh <simar@linux.com>

* feat: Add CSV template for webhook output

Signed-off-by: Simarpreet Singh <simar@linux.com>

* feat: Add XML template for webhooks

Signed-off-by: Simarpreet Singh <simar@linux.com>

* feat: Update readme.md for webhook-template

Signed-off-by: Simarpreet Singh <simar@linux.com>

* fix: Format in UTC time

Signed-off-by: Simarpreet Singh <simar@linux.com>

* feat: Initialize template once and pass around

Signed-off-by: Simarpreet Singh <simar@linux.com>

* feat: Update readme to include template usage

Signed-off-by: Simarpreet Singh <simar@linux.com>

* feat: Add --content-type flag

Signed-off-by: Simarpreet Singh <simar@linux.com>

* feat: Use current time for template payloads

Signed-off-by: Simarpreet Singh <simar@linux.com>

* fix: Log but don't error if content-type is not set

Signed-off-by: Simarpreet Singh <simar@linux.com>

* feat: Update Readme to reflect new flag usage

Signed-off-by: Simarpreet Singh <simar@linux.com>

tracee-rules/Readme.md

@@ -31,7 +31,13 @@ This will:
 
 # Integrations
 
-When a detection is made by any of the signatures, it will be printed to stdout. Using the `--webhook` flag you can post detections into an HTTP endpoint that can further relay the detection.
+When a detection is made by any of the signatures, it will be printed to stdout. Using the `--webhook` flag you can post detections into an HTTP endpoint that can further relay the detection. By default, payloads are sent as JSON to the webhook.
+
+You can also use a custom template (or use a pre-supplied one) to further tune your webhook detection output. Templates are written in the Go templating language. 
+
+When supplying a custom template, fields of the `types.Finding` event type can be used as output fields. These are available [here](https://github.com/aquasecurity/tracee/blob/28fbc66be8c9f3efa53f617a654cafe7421e8c70/tracee-rules/types/types.go#L46-L50). Some examples of custom templates are documented [here](templates/).
+
+ Custom templates can be passed in via the `--webhook-template` flag. Payload Content-Type can be specified (and is recommended) when using a custom template with the `--webhook-content-type` flag.
 
 # Rules
 Rules are discovered from the local `rules` directory (unless changed by the `--rules-dir` flag). By default, all discovered rules will be loaded unless specific rules are selected using the `--rules` flag.

tracee-rules/goldens/broken.tmpl

@@ -0,0 +1 @@
+{{ .InvalidField }}

tracee-rules/main.go

@@ -68,7 +68,7 @@ func main() {
 			if inputs == (engine.EventSources{}) {
 				return err
 			}
-			output, err := setupOutput(os.Stdout, realClock{}, c.String("webhook"))
+			output, err := setupOutput(os.Stdout, realClock{}, c.String("webhook"), c.String("webhook-template"), c.String("webhook-content-type"))
 			if err != nil {
 				return err
 			}
@@ -93,6 +93,14 @@ func main() {
 				Name:  "webhook",
 				Usage: "HTTP endpoint to call for every match",
 			},
+			&cli.StringFlag{
+				Name:  "webhook-template",
+				Usage: "path to a gotemplate for formatting webhook output",
+			},
+			&cli.StringFlag{
+				Name:  "webhook-content-type",
+				Usage: "content type of the template in use. Recommended if using --webhook-template",
+			},
 			&cli.StringSliceFlag{
 				Name:  "input-tracee",
 				Usage: "configure tracee-ebpf as input source. see '--input-tracee help' for more info",

tracee-rules/output.go

@@ -1,12 +1,15 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"log"
 	"net/http"
+	"path/filepath"
 	"strings"
+	"text/template"
 	"time"
 
 	tracee "github.com/aquasecurity/tracee/tracee-ebpf/tracee/external"
@@ -23,9 +26,15 @@ Command: %s
 Hostname: %s
 `
 
-func setupOutput(resultWriter io.Writer, clock Clock, webhook string) (chan types.Finding, error) {
+func setupOutput(resultWriter io.Writer, clock Clock, webhook string, webhookTemplate string, contentType string) (chan types.Finding, error) {
 	out := make(chan types.Finding)
-	go func() {
+
+	t, err := setupTemplate(webhookTemplate, realClock{})
+	if err != nil && webhookTemplate != "" {
+		return nil, fmt.Errorf("error preparing webhook template: %v", err)
+	}
+
+	go func(t *template.Template) {
 		for res := range out {
 			sigMetadata, err := res.Signature.GetMetadata()
 			if err != nil {
@@ -44,24 +53,59 @@ func setupOutput(resultWriter io.Writer, clock Clock, webhook string) (chan type
 			}
 
 			if webhook != "" {
-				payload, err := prepareJSONPayload(res)
-				if err != nil {
-					log.Printf("error preparing json payload for %v: %v", res, err)
-					continue
+				if err := sendToWebhook(t, res, webhook, webhookTemplate, contentType, realClock{}); err != nil {
+					log.Println(err)
 				}
-				resp, err := http.Post(webhook, "application/json", strings.NewReader(payload))
-				if err != nil {
-					log.Printf("error calling webhook for %v: %v", res, err)
-					continue
-				}
-				resp.Body.Close()
 			}
 		}
-	}()
+	}(t)
 	return out, nil
 }
 
-func prepareJSONPayload(res types.Finding) (string, error) {
+func setupTemplate(webhookTemplate string, clock Clock) (*template.Template, error) {
+	return template.New(filepath.Base(webhookTemplate)).
+		Funcs(map[string]interface{}{
+			"timeNow": func(unixTs float64) string {
+				return clock.Now().UTC().Format("2006-01-02T15:04:05Z")
+			},
+		}).ParseFiles(webhookTemplate)
+}
+
+func sendToWebhook(t *template.Template, res types.Finding, webhook string, webhookTemplate string, contentType string, clock Clock) error {
+	var payload string
+
+	switch {
+	case webhookTemplate != "":
+		if t == nil {
+			return fmt.Errorf("error writing to template: template not initialized")
+		}
+		if contentType == "" {
+			log.Println("content-type was not set for the custom template: ", webhookTemplate)
+		}
+		buf := bytes.Buffer{}
+		if err := t.Execute(&buf, res); err != nil {
+			return fmt.Errorf("error writing to the template: %v", err)
+		}
+		payload = buf.String()
+
+	default: // if no input template is specified, we send JSON by default
+		contentType = "application/json"
+		var err error
+		payload, err = prepareJSONPayload(res, clock)
+		if err != nil {
+			return fmt.Errorf("error preparing json payload: %v", err)
+		}
+	}
+
+	resp, err := http.Post(webhook, contentType, strings.NewReader(payload))
+	if err != nil {
+		return fmt.Errorf("error calling webhook %v", err)
+	}
+	_ = resp.Body.Close()
+	return nil
+}
+
+func prepareJSONPayload(res types.Finding, clock Clock) (string, error) {
 	// compatible with Falco webhook format, for easy integration with "falcosecurity/falcosidekick"
 	// https://github.com/falcosecurity/falcosidekick/blob/e6b893f612e92352ba700bed9a19f1ec2cd18260/types/types.go#L12
 	type Payload struct {
@@ -82,7 +126,7 @@ func prepareJSONPayload(res types.Finding) (string, error) {
 	payload := Payload{
 		Output:       fmt.Sprintf("Rule \"%s\" detection:\n %v", sigmeta.Name, res.Data),
 		Rule:         sigmeta.Name,
-		Time:         time.Now(),
+		Time:         clock.Now().UTC(),
 		OutputFields: fields,
 	}
 	payloadJSON, err := json.Marshal(payload)

tracee-rules/output_test.go

@@ -2,6 +2,10 @@ package main
 
 import (
 	"bytes"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
@@ -22,9 +26,14 @@ func (fakeClock) Now() time.Time {
 
 type fakeSignature struct {
 	types.Signature
+	getMetadata func() (types.SignatureMetadata, error)
 }
 
 func (f fakeSignature) GetMetadata() (types.SignatureMetadata, error) {
+	if f.getMetadata != nil {
+		return f.getMetadata()
+	}
+
 	return types.SignatureMetadata{
 		ID:          "FOO-666",
 		Name:        "foo bar signature",
@@ -41,7 +50,6 @@ func Test_setupOutput(t *testing.T) {
 		{
 			name: "happy path with tracee event",
 			inputContext: external.Event{
-				Timestamp:   12345678,
 				ProcessName: "foobar.exe",
 				HostName:    "foobar.local",
 			},
@@ -66,7 +74,7 @@ Hostname: foobar.local
 
 	for _, tc := range testCases {
 		var actualOutput bytes.Buffer
-		findingCh, err := setupOutput(&actualOutput, fakeClock{}, "")
+		findingCh, err := setupOutput(&actualOutput, fakeClock{}, "", "", "")
 		require.NoError(t, err, tc.name)
 
 		findingCh <- types.Finding{
@@ -82,3 +90,109 @@ Hostname: foobar.local
 		assert.Equal(t, tc.expectedOutput, actualOutput.String(), tc.name)
 	}
 }
+
+func Test_sendToWebhook(t *testing.T) {
+	var testCases = []struct {
+		name               string
+		inputTemplateFile  string
+		inputSignature     fakeSignature
+		inputTestServerURL string
+		contentType        string
+		expectedOutput     string
+		expectedError      string
+	}{
+		{
+			name:           "happy path, no template JSON output",
+			contentType:    "application/json",
+			expectedOutput: `{"output":"Rule \"foo bar signature\" detection:\n map[foo1:bar1, baz1 foo2:[bar2 baz2]]","rule":"foo bar signature","time":"2021-02-23T01:54:57Z","output_fields":{"value":0}}`,
+		},
+		{
+			name:        "happy path, with simple template",
+			contentType: "text/plain",
+			expectedOutput: `*** Detection ***
+Timestamp: 2021-02-23T01:54:57Z
+ProcessName: foobar.exe
+HostName: foobar.local
+`,
+			inputTemplateFile: "templates/simple.tmpl",
+		},
+		{
+			name:              "happy path, with CSV template",
+			contentType:       "text/csv",
+			expectedOutput:    `2021-02-23T01:54:57Z,foobar.exe,foobar.local`,
+			inputTemplateFile: "templates/csv.tmpl",
+		},
+		{
+			name:        "happy path, with XML template",
+			contentType: "application/xml",
+			expectedOutput: `<?xml version="1.0" encoding="UTF-8" ?>
+ <detection timestamp="2021-02-23T01:54:57Z">
+    <processname>foobar.exe</processname>
+    <hostname>foobar.local</hostname>
+ </detection>`,
+			inputTemplateFile: "templates/xml.tmpl",
+		},
+		{
+			name: "sad path, with failing GetMetadata func for sig",
+			inputSignature: fakeSignature{
+				getMetadata: func() (types.SignatureMetadata, error) {
+					return types.SignatureMetadata{}, errors.New("getMetadata failed")
+				},
+			},
+			expectedError: "error preparing json payload: getMetadata failed",
+		},
+		{
+			name:               "sad path, error reaching webhook",
+			inputTestServerURL: "foo://bad.host",
+			expectedError:      `error calling webhook Post "foo://bad.host": unsupported protocol scheme "foo"`,
+		},
+		{
+			name:              "sad path, with missing template",
+			inputTemplateFile: "invalid/template",
+			expectedError:     `error writing to template: template not initialized`,
+		},
+		{
+			name:              "sad path, with an invalid template",
+			contentType:       "application/foo",
+			inputTemplateFile: "goldens/broken.tmpl",
+			expectedError:     `error writing to the template: template: broken.tmpl:1:3: executing "broken.tmpl" at <.InvalidField>: can't evaluate field InvalidField in type types.Finding`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ts := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+				got, _ := ioutil.ReadAll(request.Body)
+				assert.Equal(t, tc.expectedOutput, string(got), tc.name)
+				assert.Equal(t, tc.contentType, request.Header.Get("content-type"), tc.name)
+			}))
+			defer ts.Close()
+
+			if tc.inputTestServerURL != "" {
+				ts.URL = tc.inputTestServerURL
+			}
+
+			inputTemplate, _ := setupTemplate(tc.inputTemplateFile, fakeClock{})
+
+			actualError := sendToWebhook(inputTemplate, types.Finding{
+				Data: map[string]interface{}{
+					"foo1": "bar1, baz1",
+					"foo2": []string{"bar2", "baz2"},
+				},
+				Context: external.Event{
+					ProcessName: "foobar.exe",
+					HostName:    "foobar.local",
+				},
+				Signature: tc.inputSignature,
+			}, ts.URL, tc.inputTemplateFile, tc.contentType, fakeClock{})
+
+			switch {
+			case tc.expectedError != "":
+				assert.EqualError(t, actualError, tc.expectedError, tc.name)
+			default:
+				assert.NoError(t, actualError, tc.name)
+			}
+		})
+
+	}
+}

tracee-rules/templates/csv.tmpl

@@ -0,0 +1 @@
+{{ .Context.Timestamp | timeNow }},{{ .Context.ProcessName }},{{ .Context.HostName }}

tracee-rules/templates/simple.tmpl

@@ -0,0 +1,4 @@
+*** Detection ***
+Timestamp: {{ .Context.Timestamp | timeNow }}
+ProcessName: {{ .Context.ProcessName }}
+HostName: {{ .Context.HostName }}

tracee-rules/templates/xml.tmpl

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+ <detection timestamp="{{.Context.Timestamp | timeNow }}">
+    <processname>{{ .Context.ProcessName }}</processname>
+    <hostname>{{ .Context.HostName }}</hostname>
+ </detection>