Skip to content

Commit

Permalink
logging: Perform filtering on arrays of strings (where possible) (#5101)
Browse files Browse the repository at this point in the history 
* logging: Perform filtering on arrays of strings (where possible)

* Add test for ip_mask filter

* Oops, need to continue when it's not an IP

* Test for invalid IPs
  • Loading branch information
@francislavoie
francislavoie committed Oct 5, 2022
1 parent 9e1d964 commit ea58d51
Show file tree
Hide file tree
Showing 2 changed files with 171 additions and 24 deletions.
83 changes: 61 additions & 22 deletions modules/logging/filters.go
View file
Expand Up @@ -23,6 +23,7 @@ import (
"net/url"
"regexp"
"strconv"
"strings"

"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
Expand Down Expand Up @@ -76,7 +77,10 @@ func hash(s string) string {
}

// HashFilter is a Caddy log field filter that
// replaces the field with the initial 4 bytes of the SHA-256 hash of the content.
// replaces the field with the initial 4 bytes
// of the SHA-256 hash of the content. Operates
// on string fields, or on arrays of strings
// where each string is hashed.
type HashFilter struct {
}

Expand All @@ -95,7 +99,13 @@ func (f *HashFilter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {

// Filter filters the input field with the replacement value.
func (f *HashFilter) Filter(in zapcore.Field) zapcore.Field {
in.String = hash(in.String)
if array, ok := in.Interface.(caddyhttp.LoggableStringArray); ok {
for i, s := range array {
array[i] = hash(s)
}
} else {
in.String = hash(in.String)
}
return in
}

Expand Down Expand Up @@ -131,7 +141,10 @@ func (f *ReplaceFilter) Filter(in zapcore.Field) zapcore.Field {
}

// IPMaskFilter is a Caddy log field filter that
// masks IP addresses.
// masks IP addresses in a string, or in an array
// of strings. The string may be a comma separated
// list of IP addresses, where all of the values
// will be masked.
type IPMaskFilter struct {
// The IPv4 mask, as an subnet size CIDR.
IPv4MaskRaw int `json:"ipv4_cidr,omitempty"`
Expand Down Expand Up @@ -205,27 +218,45 @@ func (m *IPMaskFilter) Provision(ctx caddy.Context) error {

// Filter filters the input field.
func (m IPMaskFilter) Filter(in zapcore.Field) zapcore.Field {
host, port, err := net.SplitHostPort(in.String)
if err != nil {
host = in.String // assume whole thing was IP address
}
ipAddr := net.ParseIP(host)
if ipAddr == nil {
return in
}
mask := m.v4Mask
if ipAddr.To4() == nil {
mask = m.v6Mask
}
masked := ipAddr.Mask(mask)
if port == "" {
in.String = masked.String()
if array, ok := in.Interface.(caddyhttp.LoggableStringArray); ok {
for i, s := range array {
array[i] = m.mask(s)
}
} else {
in.String = net.JoinHostPort(masked.String(), port)
in.String = m.mask(in.String)
}

return in
}

func (m IPMaskFilter) mask(s string) string {
output := ""
for _, value := range strings.Split(s, ",") {
value = strings.TrimSpace(value)
host, port, err := net.SplitHostPort(value)
if err != nil {
host = value // assume whole thing was IP address
}
ipAddr := net.ParseIP(host)
if ipAddr == nil {
output += value + ", "
continue
}
mask := m.v4Mask
if ipAddr.To4() == nil {
mask = m.v6Mask
}
masked := ipAddr.Mask(mask)
if port == "" {
output += masked.String() + ", "
continue
}

output += net.JoinHostPort(masked.String(), port) + ", "
}
return strings.TrimSuffix(output, ", ")
}

type filterAction string

const (
Expand Down Expand Up @@ -499,7 +530,10 @@ OUTER:
}

// RegexpFilter is a Caddy log field filter that
// replaces the field matching the provided regexp with the indicated string.
// replaces the field matching the provided regexp
// with the indicated string. If the field is an
// array of strings, each of them will have the
// regexp replacement applied.
type RegexpFilter struct {
// The regular expression pattern defining what to replace.
RawRegexp string `json:"regexp,omitempty"`
Expand Down Expand Up @@ -545,7 +579,13 @@ func (m *RegexpFilter) Provision(ctx caddy.Context) error {

// Filter filters the input field with the replacement value if it matches the regexp.
func (f *RegexpFilter) Filter(in zapcore.Field) zapcore.Field {
in.String = f.regexp.ReplaceAllString(in.String, f.Value)
if array, ok := in.Interface.(caddyhttp.LoggableStringArray); ok {
for i, s := range array {
array[i] = f.regexp.ReplaceAllString(s, f.Value)
}
} else {
in.String = f.regexp.ReplaceAllString(in.String, f.Value)
}

return in
}
Expand Down Expand Up @@ -576,7 +616,6 @@ func (f *RenameFilter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {

// Filter renames the input field with the replacement name.
func (f *RenameFilter) Filter(in zapcore.Field) zapcore.Field {
in.Type = zapcore.StringType
in.Key = f.Name
return in
}
Expand Down
112 changes: 110 additions & 2 deletions modules/logging/filters_test.go
View file
Expand Up @@ -8,6 +8,81 @@ import (
"go.uber.org/zap/zapcore"
)

func TestIPMaskSingleValue(t *testing.T) {
f := IPMaskFilter{IPv4MaskRaw: 16, IPv6MaskRaw: 32}
f.Provision(caddy.Context{})

out := f.Filter(zapcore.Field{String: "255.255.255.255"})
if out.String != "255.255.0.0" {
t.Fatalf("field has not been filtered: %s", out.String)
}

out = f.Filter(zapcore.Field{String: "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"})
if out.String != "ffff:ffff::" {
t.Fatalf("field has not been filtered: %s", out.String)
}

out = f.Filter(zapcore.Field{String: "not-an-ip"})
if out.String != "not-an-ip" {
t.Fatalf("field has been filtered: %s", out.String)
}
}

func TestIPMaskCommaValue(t *testing.T) {
f := IPMaskFilter{IPv4MaskRaw: 16, IPv6MaskRaw: 32}
f.Provision(caddy.Context{})

out := f.Filter(zapcore.Field{String: "255.255.255.255, 244.244.244.244"})
if out.String != "255.255.0.0, 244.244.0.0" {
t.Fatalf("field has not been filtered: %s", out.String)
}

out = f.Filter(zapcore.Field{String: "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff, ff00:ffff:ffff:ffff:ffff:ffff:ffff:ffff"})
if out.String != "ffff:ffff::, ff00:ffff::" {
t.Fatalf("field has not been filtered: %s", out.String)
}

out = f.Filter(zapcore.Field{String: "not-an-ip, 255.255.255.255"})
if out.String != "not-an-ip, 255.255.0.0" {
t.Fatalf("field has not been filtered: %s", out.String)
}
}

func TestIPMaskMultiValue(t *testing.T) {
f := IPMaskFilter{IPv4MaskRaw: 16, IPv6MaskRaw: 32}
f.Provision(caddy.Context{})

out := f.Filter(zapcore.Field{Interface: caddyhttp.LoggableStringArray{
"255.255.255.255",
"244.244.244.244",
}})
arr, ok := out.Interface.(caddyhttp.LoggableStringArray)
if !ok {
t.Fatalf("field is wrong type: %T", out.Integer)
}
if arr[0] != "255.255.0.0" {
t.Fatalf("field entry 0 has not been filtered: %s", arr[0])
}
if arr[1] != "244.244.0.0" {
t.Fatalf("field entry 1 has not been filtered: %s", arr[1])
}

out = f.Filter(zapcore.Field{Interface: caddyhttp.LoggableStringArray{
"ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
"ff00:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
}})
arr, ok = out.Interface.(caddyhttp.LoggableStringArray)
if !ok {
t.Fatalf("field is wrong type: %T", out.Integer)
}
if arr[0] != "ffff:ffff::" {
t.Fatalf("field entry 0 has not been filtered: %s", arr[0])
}
if arr[1] != "ff00:ffff::" {
t.Fatalf("field entry 1 has not been filtered: %s", arr[1])
}
}

func TestQueryFilter(t *testing.T) {
f := QueryFilter{[]queryFilterAction{
{replaceAction, "foo", "REDACTED"},
Expand Down Expand Up @@ -78,7 +153,7 @@ func TestValidateCookieFilter(t *testing.T) {
}
}

func TestRegexpFilter(t *testing.T) {
func TestRegexpFilterSingleValue(t *testing.T) {
f := RegexpFilter{RawRegexp: `secret`, Value: "REDACTED"}
f.Provision(caddy.Context{})

Expand All @@ -88,11 +163,44 @@ func TestRegexpFilter(t *testing.T) {
}
}

func TestHashFilter(t *testing.T) {
func TestRegexpFilterMultiValue(t *testing.T) {
f := RegexpFilter{RawRegexp: `secret`, Value: "REDACTED"}
f.Provision(caddy.Context{})

out := f.Filter(zapcore.Field{Interface: caddyhttp.LoggableStringArray{"foo-secret-bar", "bar-secret-foo"}})
arr, ok := out.Interface.(caddyhttp.LoggableStringArray)
if !ok {
t.Fatalf("field is wrong type: %T", out.Integer)
}
if arr[0] != "foo-REDACTED-bar" {
t.Fatalf("field entry 0 has not been filtered: %s", arr[0])
}
if arr[1] != "bar-REDACTED-foo" {
t.Fatalf("field entry 1 has not been filtered: %s", arr[1])
}
}

func TestHashFilterSingleValue(t *testing.T) {
f := HashFilter{}

out := f.Filter(zapcore.Field{String: "foo"})
if out.String != "2c26b46b" {
t.Fatalf("field has not been filtered: %s", out.String)
}
}

func TestHashFilterMultiValue(t *testing.T) {
f := HashFilter{}

out := f.Filter(zapcore.Field{Interface: caddyhttp.LoggableStringArray{"foo", "bar"}})
arr, ok := out.Interface.(caddyhttp.LoggableStringArray)
if !ok {
t.Fatalf("field is wrong type: %T", out.Integer)
}
if arr[0] != "2c26b46b" {
t.Fatalf("field entry 0 has not been filtered: %s", arr[0])
}
if arr[1] != "fcde2b2e" {
t.Fatalf("field entry 1 has not been filtered: %s", arr[1])
}
}

0 comments on commit ea58d51

Please sign in to comment.