diff --git a/README.md b/README.md
index 1acb17e0..1dabe59d 100644
--- a/README.md
+++ b/README.md
@@ -46,7 +46,7 @@ kubectl create secret generic honeycomb --from-literal=api-key=$HONEYCOMB_API_KE
The network agent can be configured using the following environment variables.
| Environment Variable | Description | Default | Required? |
-| ------------------------- | ---------------------------------------------------------------------------------------- | -------------------------- | --------- |
+|---------------------------|------------------------------------------------------------------------------------------|----------------------------|-----------|
| `HONEYCOMB_API_KEY` | The Honeycomb API key used when sending events | `` (empty) | **Yes** |
| `HONEYCOMB_API_ENDPOINT` | The endpoint to send events to | `https://api.honeycomb.io` | No |
| `HONEYCOMB_DATASET` | Dataset where network events are stored | `hny-network-agent` | No |
@@ -54,6 +54,9 @@ The network agent can be configured using the following environment variables.
| `LOG_LEVEL` | The log level to use when printing logs to console | `INFO` | No |
| `DEBUG` | Runs the agent in debug mode including enabling a profiling endpoint using Debug Address | `false` | No |
| `DEBUG_ADDRESS` | The endpoint to listen to when running the profile endpoint | `localhost:6060` | No |
+| `HTTP_HEADERS` | Case-sensitive, comma separated list of headers to be recorded from requests/responses† | `User-Agent` | No |
+
+†: When providing an overide of a list of values, you must provide all values including any defaults.
### Run
@@ -67,12 +70,12 @@ Alternative options for configuration and running can be found in [Deploying the
## Supported Platforms
-| Platform | Supported |
-| ---------------------------------------------------------------------| ------------------------------------- |
-| [AKS](https://azure.microsoft.com/en-gb/products/kubernetes-service) | Supported ✅ |
-| [EKS](https://aws.amazon.com/eks/) | Self-managed hosts ✅
Fargate ❌ |
-| [GKE](https://cloud.google.com/kubernetes-engine) | Standard cluster ✅
AutoPilot ❌ |
-| Self-hosted | Ubuntu ✅ |
+| Platform | Supported |
+|----------------------------------------------------------------------|-------------------------------------|
+| [AKS](https://azure.microsoft.com/en-gb/products/kubernetes-service) | Supported ✅ |
+| [EKS](https://aws.amazon.com/eks/) | Self-managed hosts ✅
Fargate ❌ |
+| [GKE](https://cloud.google.com/kubernetes-engine) | Standard cluster ✅
AutoPilot ❌ |
+| Self-hosted | Ubuntu ✅ |
### Requirements
diff --git a/assemblers/http_parser.go b/assemblers/http_parser.go
index 0c275f09..5cff7722 100644
--- a/assemblers/http_parser.go
+++ b/assemblers/http_parser.go
@@ -8,12 +8,14 @@ import (
// httpParser parses HTTP requests and responses
type httpParser struct {
- matcher *httpMatcher
+ matcher *httpMatcher
+ headersToExtract []string
}
-func newHttpParser() *httpParser {
+func newHttpParser(headersToExtract []string) *httpParser {
return &httpParser{
- matcher: newRequestResponseMatcher(),
+ matcher: newRequestResponseMatcher(),
+ headersToExtract: headersToExtract,
}
}
@@ -28,7 +30,7 @@ func (parser *httpParser) parse(stream *tcpStream, requestId int64, timestamp ti
return false, err
}
// We only care about a few headers, so recreate the header with just the ones we need
- req.Header = extractHeaders(req.Header)
+ req.Header = parser.extractHeaders(req.Header)
// We don't need the body, so just close it if set
if req.Body != nil {
req.Body.Close()
@@ -54,7 +56,7 @@ func (parser *httpParser) parse(stream *tcpStream, requestId int64, timestamp ti
return false, err
}
// We only care about a few headers, so recreate the header with just the ones we need
- res.Header = extractHeaders(res.Header)
+ res.Header = parser.extractHeaders(res.Header)
// We don't need the body, so just close it if set
if res.Body != nil {
res.Body.Close()
@@ -78,19 +80,15 @@ func (parser *httpParser) parse(stream *tcpStream, requestId int64, timestamp ti
return true, nil
}
-var headersToExtract = []string{
- "User-Agent",
-}
-
// extractHeaders returns a new http.Header object with only specified headers from the original.
// The original request/response header contains a lot of stuff we don't really care about
// and stays in memory until the request/response pair is processed
-func extractHeaders(header http.Header) http.Header {
+func (parser *httpParser) extractHeaders(header http.Header) http.Header {
cleanHeader := http.Header{}
if header == nil {
return cleanHeader
}
- for _, headerName := range headersToExtract {
+ for _, headerName := range parser.headersToExtract {
if headerValue := header.Get(headerName); headerValue != "" {
cleanHeader.Set(headerName, headerValue)
}
diff --git a/assemblers/http_parser_test.go b/assemblers/http_parser_test.go
index ef3b1549..fce52cfd 100644
--- a/assemblers/http_parser_test.go
+++ b/assemblers/http_parser_test.go
@@ -9,37 +9,52 @@ import (
func TestExtractHeader(t *testing.T) {
testCases := []struct {
- name string
- header http.Header
- expected http.Header
+ name string
+ headersToExtract []string
+ header http.Header
+ expected http.Header
}{
{
- name: "nil header",
- header: nil,
- expected: http.Header{},
+ name: "nil header",
+ headersToExtract: nil,
+ header: nil,
+ expected: http.Header{},
},
{
- name: "empty header",
- header: http.Header{},
- expected: http.Header{},
+ name: "empty header",
+ headersToExtract: nil,
+ header: http.Header{},
+ expected: http.Header{},
},
{
- name: "only extracts headers we want to keep",
+ name: "only extracts headers we want to keep",
+ headersToExtract: []string{"User-Agent", "X-Test"},
header: http.Header{
"Accept": []string{"test"},
"Host": []string{"test"},
"Cookie": []string{"test"},
"User-Agent": []string{"test"},
+ "X-Test": []string{"test"},
},
expected: http.Header{
"User-Agent": []string{"test"},
+ "X-Test": []string{"test"},
},
},
+ {
+ name: "header names are case-sensitive",
+ headersToExtract: []string{"X-TEST"},
+ header: http.Header{
+ "x-test": []string{"test"},
+ },
+ expected: http.Header{},
+ },
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
- result := extractHeaders(tc.header)
+ parser := newHttpParser(tc.headersToExtract)
+ result := parser.extractHeaders(tc.header)
assert.Equal(t, tc.expected, result)
})
}
diff --git a/assemblers/tcp_stream.go b/assemblers/tcp_stream.go
index 3458ce19..adc18492 100644
--- a/assemblers/tcp_stream.go
+++ b/assemblers/tcp_stream.go
@@ -51,7 +51,7 @@ func NewTcpStream(net gopacket.Flow, transport gopacket.Flow, config config.Conf
dstPort: transport.Dst().String(),
buffer: bufio.NewReader(bytes.NewReader(nil)),
parsers: []parser{
- newHttpParser(),
+ newHttpParser(config.HTTPHeadersToExtract),
},
}
}
diff --git a/config/config.go b/config/config.go
index b0e75f0b..e0d6ac25 100644
--- a/config/config.go
+++ b/config/config.go
@@ -109,6 +109,9 @@ type Config struct {
// Include the request URL in the event.
IncludeRequestURL bool
+
+ // The list of HTTP headers to extract from a HTTP request/response.
+ HTTPHeadersToExtract []string
}
// NewConfig returns a new Config struct.
@@ -145,6 +148,7 @@ func NewConfig() Config {
AgentPodName: utils.LookupEnvOrString("AGENT_POD_NAME", ""),
AdditionalAttributes: utils.LookupEnvAsStringMap("ADDITIONAL_ATTRIBUTES"),
IncludeRequestURL: utils.LookupEnvOrBool("INCLUDE_REQUEST_URL", true),
+ HTTPHeadersToExtract: getHTTPHeadersToExtract(),
}
}
@@ -236,3 +240,16 @@ func (c *Config) Validate() error {
// returns nil if no errors in slice
return errors.Join(e...)
}
+
+var defaultHeadersToExtract = []string{
+ "User-Agent",
+}
+
+// getHTTPHeadersToExtract returns the list of HTTP headers to extract from a HTTP request/response
+// based on a user-defined list in HTTP_HEADERS, or the default headers if no list is given.
+func getHTTPHeadersToExtract() []string {
+ if headers, found := utils.LookupEnvAsStringSlice("HTTP_HEADERS"); found {
+ return headers
+ }
+ return defaultHeadersToExtract
+}
diff --git a/config/config_test.go b/config/config_test.go
index 3636ef0c..ecd893b6 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -66,6 +66,7 @@ func TestEnvVars(t *testing.T) {
t.Setenv("AGENT_POD_NAME", "pod_name")
t.Setenv("ADDITIONAL_ATTRIBUTES", "key1=value1,key2=value2")
t.Setenv("INCLUDE_REQUEST_URL", "false")
+ t.Setenv("HTTP_HEADERS", "header1,header2")
config := NewConfig()
assert.Equal(t, "1234567890123456789012", config.APIKey)
@@ -82,6 +83,14 @@ func TestEnvVars(t *testing.T) {
assert.Equal(t, "pod_name", config.AgentPodName)
assert.Equal(t, map[string]string{"key1": "value1", "key2": "value2"}, config.AdditionalAttributes)
assert.Equal(t, false, config.IncludeRequestURL)
+ assert.Equal(t, []string{"header1", "header2"}, config.HTTPHeadersToExtract)
+}
+
+func TestEmptyHeadersEnvVar(t *testing.T) {
+ t.Setenv("HTTP_HEADERS", "")
+
+ config := NewConfig()
+ assert.Equal(t, []string{}, config.HTTPHeadersToExtract)
}
func TestEnvVarsDefault(t *testing.T) {
@@ -106,6 +115,7 @@ func TestEnvVarsDefault(t *testing.T) {
assert.Equal(t, "", config.AgentPodName)
assert.Equal(t, map[string]string{}, config.AdditionalAttributes)
assert.Equal(t, true, config.IncludeRequestURL)
+ assert.Equal(t, []string{"User-Agent"}, config.HTTPHeadersToExtract)
}
func Test_Config_buildBpfFilter(t *testing.T) {
diff --git a/utils/env.go b/utils/env.go
index 7c5c9c47..22b0aa19 100644
--- a/utils/env.go
+++ b/utils/env.go
@@ -41,3 +41,21 @@ func LookupEnvAsStringMap(key string) map[string]string {
}
return values
}
+
+// LookupEnvAsStringSlice returns a slice of strings from the environment variable with the given key
+// and a boolean indicating if the key was present
+// values are comma separated
+// Example: value1,value2,value3
+func LookupEnvAsStringSlice(key string) ([]string, bool) {
+ values := []string{}
+ env, found := os.LookupEnv(key)
+ if !found {
+ return values, false
+ }
+ for _, value := range strings.Split(env, ",") {
+ if value != "" {
+ values = append(values, value)
+ }
+ }
+ return values, true
+}