enhancement: Add audit log filtering to Hub backend (#2073)

Add support to specify filters for certain Audit Log fields.
The values are removed prior to exporting to the Hub backend.

The following fields are included:
* metadata
* peer
* method: `input(s)` and `output(s)` (Decision Logs only)

We support a subset of JSONPath operations, as follows:

- dot notation: `foo.bar.baz`
- or bracket-notation: `['foo']['bar']['baz]`
- or combinations thereof

`bar` or `baz` above can be map keys, nested messages or structs.

We support list indexing with Ints or wildcards:
- foo.bar[0]
- foo.bar[*]

Wildcards can also operate on member names as a match-all. E.g `foo[*].baz`
will match both `baz` values in the pseudo-object below:

```
{
  'foo': {
    'pow': {
        'baz',
    },
    'bosh': {
        'baz',
    },
  }
}
```
---------

Signed-off-by: Sam Lock <sam@swlock.co.uk>
Co-authored-by: Charith Ellawala <charithe@users.noreply.github.com>

docs/modules/configuration/partials/fullconfiguration.adoc

@@ -24,6 +24,16 @@ audit:
       flushInterval: 1s 
       gcInterval: 60s 
       maxBatchSize: 32 
+    mask: # Mask defines a list of attributes to exclude from the audit logs, specified as lists of JSONPaths
+      checkResources: 
+        - inputs[*].principal.attr.foo
+        - inputs[*].auxData
+        - outputs 
+      metadata: ['authorization'] 
+      peer: 
+        - address
+        - forwarded_for 
+      planResources: ['input.principal.attr.nestedMap.foo'] 
     retentionPeriod: 168h # How long to keep records for
     storagePath: /path/to/dir # Path to store the data
   kafka:

go.mod

@@ -54,6 +54,7 @@ require (
 	github.com/microsoft/go-mssqldb v1.7.0
 	github.com/minio/minio-go/v7 v7.0.69
 	github.com/nlepage/go-tarfs v1.2.1
+	github.com/ohler55/ojg v1.21.4
 	github.com/oklog/ulid/v2 v2.1.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/ory/dockertest/v3 v3.10.0

go.sum

@@ -579,6 +579,8 @@ github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJm
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nlepage/go-tarfs v1.2.1 h1:o37+JPA+ajllGKSPfy5+YpsNHDjZnAoyfvf5GsUa+Ks=
 github.com/nlepage/go-tarfs v1.2.1/go.mod h1:rno18mpMy9aEH1IiJVftFsqPyIpwqSUiAOpJYjlV2NA=
+github.com/ohler55/ojg v1.21.4 h1:2iWyz/xExx0XySVIxR9kWFxIdsLNrpWLrKuAcs5aOZU=
+github.com/ohler55/ojg v1.21.4/go.mod h1:gQhDVpQLqrmnd2eqGAvJtn+NfKoYJbe/A4Sj3/Vro4o=
 github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
 github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=

internal/audit/hub/conf.go

@@ -32,7 +32,9 @@ var (
 )
 
 type Conf struct {
-	Ingest     IngestConf `yaml:"ingest" conf:",ignore"`
+	Ingest IngestConf `yaml:"ingest" conf:",ignore"`
+	// Mask defines a list of attributes to exclude from the audit logs, specified as lists of JSONPaths
+	Mask       MaskConf `yaml:"mask"`
 	local.Conf `yaml:",inline"`
 }
 
@@ -51,6 +53,13 @@ type IngestConf struct {
 	NumGoRoutines uint `yaml:"numGoRoutines" conf:",example=8"`
 }
 
+type MaskConf struct {
+	Peer           []string `yaml:"peer" conf:",example=\n    - address\n    - forwarded_for"`
+	Metadata       []string `yaml:"metadata" conf:",example=['authorization']"`
+	CheckResources []string `yaml:"checkResources" conf:",example=\n    - inputs[*].principal.attr.foo\n    - inputs[*].auxData\n    - outputs"`
+	PlanResources  []string `yaml:"planResources" conf:",example=['input.principal.attr.nestedMap.foo']"`
+}
+
 func (c *Conf) Key() string {
 	return confKey
 }