Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions containers/container.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package containers
import (
"io"
"log"
"net/http"
"os"
"sort"
"strings"
Expand Down Expand Up @@ -743,11 +744,11 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R
case l7.ProtocolHTTP:
stats.observe(r.Status.Http(), "", r.Duration)
payload := ""
headers := ""
method := ""
uri := ""
response := ""
host := ""
headers := http.Header{}
req, err := l7.ParseHTTPRequest(r.Payload)
if err != nil {
log.Printf("Failed to parse payload %s, %q", err, string(r.Payload))
Expand All @@ -759,10 +760,6 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R
base64String := base64.StdEncoding.EncodeToString(body)
payload = string(base64String)
}
if req != nil && req.Header != nil {
headersStr := l7.ConvertHeadersToString(req.Header)
headers = base64.StdEncoding.EncodeToString([]byte(headersStr))
}
method = req.Method
if utf8.ValidString(req.URL.Path) {
uri = req.URL.Path
Expand All @@ -772,6 +769,9 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R
uri = string([]rune(req.URL.Path))
}
host = req.Host
if req.Header != nil {
headers = req.Header
}
}
if r.Response != nil {
response = base64.StdEncoding.EncodeToString(r.Response)
Expand Down
2 changes: 1 addition & 1 deletion containers/l7.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func (s L7Stats) get(protocol l7.Protocol, key common.DestinationKey, r *l7.Requ
labels = append(labels, "method")
case l7.ProtocolHTTP:
method, path := l7.ParseHttp(r.Payload)
if ValidUtf8(r.Payload) {
if ValidUtf8([]byte(path)) {
constLabels["path"] = path
} else {
log.Printf("Failed to parse path %s", path)
Expand Down
8 changes: 5 additions & 3 deletions ebpftracer/l7/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package l7

import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"io"
Expand Down Expand Up @@ -129,7 +130,7 @@ func ParseHostFromHttpRequest(input string) (string, error) {
return host, nil
}

func ConvertHeadersToString(headers http.Header) string {
func ConvertHeadersToBase64String(headers http.Header) string {
headerMap := make(map[string][]string)
for key, values := range headers {
headerMap[key] = values
Expand All @@ -138,13 +139,14 @@ func ConvertHeadersToString(headers http.Header) string {
if err != nil {
return ""
}
return string(jsonString)
header64 := base64.StdEncoding.EncodeToString([]byte(jsonString))
return string(header64)
}

func SanitizeString(value string) string {
if !*flags.SanitizeHeaders {

return value
}
return strings.Repeat("*", len(value))
return strings.Repeat("*", 5)
}
18 changes: 9 additions & 9 deletions ebpftracer/l7/l7_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,13 @@ func TestParseHost(t *testing.T) {
assert.Equal(t, "/1", req.URL.Path)
assert.Equal(t, "127.0.0.1", req.Host)
assert.Equal(t, "HEAD", req.Method)
assert.Equal(t, "****", req.Header.Get("Authorization"))
assert.Equal(t, "****", req.Header.Get("Cookie"))
assert.Equal(t, "*****", req.Header.Get("Authorization"))
assert.Equal(t, "*****", req.Header.Get("Cookie"))
body, err := io.ReadAll(req.Body)
assert.Nil(t, nil, err)
assert.Equal(t, "xzxxxxxxzx", string(body))

headers := ConvertHeadersToString(req.Header)
headers := ConvertHeadersToBase64String(req.Header)
assert.NotNil(t, headers)

requestString = "POST / HTTP/1.1\r\nHost: ec2.us-east-1.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.50.10 (go1.21.6; linux; amd64) karpenter.sh-v0.34.0\r\nContent-Length: 422\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ASIAUCTZOIG67J6IJFOY/20240408/us-east-1/ec2/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-security-token, Signature=3d7eadceec976dd17fbdcc07ae43a75264baf179fec70c6aa2be7411fd552e70\r\nContent-Type: application/x-www-form-urlencoded; charset=utf-8\r\nX-Amz-Date: 20240408T094330Z\r\nX-Amz-Security-Token: IQoJb3JpZ2luX2VjEML//////////wEaCXVzLWVhc3QtMSJHMEUCIB8pKvU76WFQ99xqZH8LQXhOBfCP/lhnMSkmIp1LcvLEAiEAk3XkJCPMFGQC4KqMk92boxaeNHiLvbinmW9AQgEGnBcq/gQI6///////////ARABGgwyODA1MDEzMDU3ODkiDJA/sBIYp1t0z9szDirSBPDDQ+mKiBwDvCwqZ/HH9wSh9U6WKYjh0EPX9i3cHE46oee4CAUoQwEmDy9xZp02UZcpOjiDF6iaKyOvi93+LoiwJVBNi53o/PnMoStfI4OmG4Rc61YYcPc6t6SSXeBEGCzeeesWAGBDJ6GE2PWYYQAKS3YrpxYE+UKdb4hxEBO5t8RgEO/ooiHehFC/5VSmjMeMfT/XMdmAxBplX/wCMoz2vMcpkCi3fCsX0RNjxT7bAO/D2jOUrIX8UjM38h2VGNliSfCl8B3HbCETxYiz66AhMGabknMpJ8Dcotm\x00"
Expand All @@ -118,7 +118,7 @@ func TestParseHost(t *testing.T) {
assert.Equal(t, "ec2.us-east-1.amazonaws.com", req.Host)
assert.Equal(t, "POST", req.Method)

headers = ConvertHeadersToString(req.Header)
headers = ConvertHeadersToBase64String(req.Header)
assert.NotNil(t, headers)

requestString = "POST / HTTP/1.1\r\nHost: ec2.us-east-1.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.50.10 (go1.21.6; linux; amd64) karpenter.sh-v0.34.0\r\nContent-Length: 422\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ASIAUCTZOIG67J6IJFOY/20240408/us-east-1/ec2/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-security-token, Signature=3d7eadceec976dd17fbdcc07ae43a75264baf179fec70c6aa2be7411fd552e70\r\nContent-Type: application/x-www-form-urlencoded; charset=utf-8\r\nX-Amz-Date: 20240408T094330Z\r\nX-Amz-Security-Token: IQoJb3JpZ2luX2VjEML//////////wEaCXVzLWVhc3QtMSJHMEUCIB8pKvU76WFQ99xqZH8LQXhOBfCP/lhnMSkmIp1LcvLEAiEAk3XkJCPMFGQC4KqMk92boxaeNHiLvbinmW9AQgEGnBcq/gQI6///////////ARABGgwyODA1MDEzMDU3ODkiDJA/sBIYp1t0z9szDirSBPDDQ+mKiBwDvCwqZ/HH9wSh9U6WKYjh0EPX9i3cHE46oee4CAUoQwEmDy9xZp02UZcpOjiDF6iaKyOvi93+LoiwJVBNi53o/PnMoStfI4OmG4Rc61YYcPc6t6SSXeBEGCzeeesWAGBDJ6GE2PWYYQAKS3YrpxYE+UKdb4hxEBO5t8RgEO/ooiHehFC/5VSmjMeMfT/XMdmAxBplX/wCMoz2vMcpkCi3fCsX0RNjxT7bAO/D2jOUrIX8UjM38h2VGNliSfCl8B3HbCETxYiz66AhMGabknMpJ8Dcotm\x00"
Expand All @@ -128,7 +128,7 @@ func TestParseHost(t *testing.T) {
assert.Equal(t, "ec2.us-east-1.amazonaws.com", req.Host)
assert.Equal(t, "POST", req.Method)

headers = ConvertHeadersToString(req.Header)
headers = ConvertHeadersToBase64String(req.Header)
assert.NotNil(t, headers)

requestString = "GET /api/v1/namespaces/nudgebee-agent/pods/nudgebee-agent-pgnxh/log?container=node-agent&previous=False&tailLines=1000&timestamps=True HTTP/1.1\r\nHost: 10.100.0.1\r\nAccept-Encoding: identity\r\nAccept: application/json\r\nUser-Agent: OpenAPI-Generator/26.1.0/python\r\nauthorization: bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjNhZTFlNjlkN2I1ZmFkNzhhNTg2YTVjN2JmMDNjOGU3YjUzNGI4N2MifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjIl0sImV4cCI6MTc0NDEwMjY0OSwiaWF0IjoxNzEyNTY2NjQ5LCJpc3MiOiJodHRwczovL29pZGMuZWtzLnVzLWVhc3QtMS5hbWF6b25hd3MuY29tL2lkL0I4RkU4OUQ4MjdEN0Q2RTE0REIyMUMzRkIwQTk1Q0E3Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJudWRnZWJlZS1hZ2VudCIsInBvZCI6eyJuYW1lIjoibnVkZ2ViZWUtYWdlbnQtcnVubmVyLTZiZmY2NmQ3YjgtcWttbmwiLCJ1aWQiOiJiN2M4ZGM2NC04Y2Q0LTRiMWUtODg1Zi00YWMzMWE4NDYwMWIifSwic2VydmljZWFjY291bnQiOnsibmFtZSI6Im51ZGdlYmVlLWFnZW50LXJ1bm5lci1zZXJ2aWNlLWFjY291bnQiLCJ1aWQiOiJjODUzMDc2YS1hODVkLTQzYzItOWU3NS00MTk5ODIzMTVjOGIifSwid2FybmFmdGVyIjoxNzEyNTcwMjU2fSwibmJmIjoxNzEyNTY2NjQ5LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6b\x00"
Expand All @@ -138,7 +138,7 @@ func TestParseHost(t *testing.T) {
assert.Equal(t, "10.100.0.1", req.Host)
assert.Equal(t, "GET", req.Method)

headers = ConvertHeadersToString(req.Header)
headers = ConvertHeadersToBase64String(req.Header)
assert.NotNil(t, headers)

requestString = "PUT /nudgebee-dev-loki-logs/fake/ece64b2028d30d33/18ebcc10d27%3A18ebcfd01d3%3A3377d9aa HTTP/1.1\r\nHost: s3.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.44.315 (go1.21.3; linux; amd64)\r\nContent-Length: 14183\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ASIAUCTZOIG62SVE2F5J/20240408/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class, Signature=a4db153ba7127721c3a5efbbab1a0e194e473d23147a472c92ff672d9dff160c\r\nContent-Md5: VBSxwLVjCvMeC+z4V86k8A==\r\nX-Amz-Content-Sha256: fe19171172133fadb236229e71a8f60169fc5268da749b1eeeeb3396e7d90f64\r\nX-Amz-Date: 20240408T094454Z\r\nX-Amz-Security-Token: IQoJb3JpZ2luX2VjEL3//////////wEaCXVzLWVhc3QtMSJGMEQCIAvNhx/LC+nIkqUC+rRD3MOFuxr39bc+qmRwVPljywxuAiA/8bmpr+kmRy7zXBzBQ7eyayLftqLutyGrzu8TIZWVbyrGBQjm//////////8BEAEaDDI4MDUwMTMwNTc4OSIMRsng8zavN9ilo6+0KpoFc/pdelkZBjWQle/CNG4XKcnd9WFbu87JAB3xa1P97vfIBjhqpmHoPk4V38r21Vepl0NjeUnWahJA6NnOWW2mdhRv51+WGUmKSHz1j6NCRcC8axDXs65jJzF6WgqCGZGNa0aiUQP7oIGGV8\x00"
Expand All @@ -148,7 +148,7 @@ func TestParseHost(t *testing.T) {
assert.Equal(t, "s3.amazonaws.com", req.Host)
assert.Equal(t, "PUT", req.Method)

headers = ConvertHeadersToString(req.Header)
headers = ConvertHeadersToBase64String(req.Header)
assert.NotNil(t, headers)

requestString = "POST /api/v1/write HTTP/1.1\r\nHost: vmsingle-victoria-victoria-metrics-k8s-stack.victoria.svc:8429\r\nUser-Agent: vmagent\r\nContent-Length: 282715\r\nContent-Encoding: snappy\r\nContent-Type: application/x-protobuf\r\nX-Prometheus-Remote-Write-Version: 0.1.0\r\nAccept-Encoding: gzip\r\n\r\n\x91\x9b\xdc\x01\xb0\n\xe4\f\n$\n\b__name__\x12\x18container_memory_failcnt\n$\n\t\x15\x1c\xd0\x12\x17nudgebee-agent-opencost\n;\n\x05image\x122quay.io/kubecost1\x15\n\x00-\x01*X-model:prod-1.108.0\n\x1b\n\t\x01\x87\x18space\x12\x0e6c\x00 \n/\n\x03pod\x12(6\x17\x00\x15z\xd8-55b677fb47-87mz4\n \n\x17beta_kubernetes_io_arch\x12\x05amd64\n.\n J\"\x00pinstance_type\x12\nm5ad.large\n\x1e\n\x15J0\x00\xf0Cos\x12\x05linux\n&\n\x1eeks_amazonaws_com_capacityType\x12\x04SPOT\n5\n(failure_domain_JW\x00Pregion\x12\tus-east-1\n4\n&\x867\x00\x14zone\x12\n\x155\x10a\n(\n\b\x11Ҙ\x12\x1cip-172-31-0-236.ec2.internal\n\x0e\n\x03job\x12\a!1 let\n=\n\x19k8%5\xf0<cloud_provider_aws\x12 ed58b207ac78b478c91e5b775a4f8540\n(\n#karpe\x01_\x00_\x01I!\x12\x11\x8b@_category\x12\x01m\n#\n\x1ekj*\x00 pu\x12\x012\nC\n:j%\x00\xa8encryption_in_transit_supported\x12\x05false\n)\n!kfj\x00\x1cfamily\x12\x04!\xf3\f\n*\n%jp\x00\x14genera\x01p\x10\x12\x015\n.r,\x00Hhypervisor\x12\x05nitro\n+r0\x008local_nvme\x12\x0275\nv\xb4\x00i\xa6$\x12\x048192\n3\n,j\xb4\x00Pnetwork_bandwidth\x12\x0375!\xa8\x00\x1fj5\x00\x14size\x12\x05i\x00\b\"\n\x1a\x19*\x04shU\xdbi* \x04spot\n \n\x182$\x00Pinitializ\x00\x12@3c"
Expand All @@ -158,7 +158,7 @@ func TestParseHost(t *testing.T) {
assert.Equal(t, "vmsingle-victoria-victoria-metrics-k8s-stack.victoria.svc:8429", req.Host)
assert.Equal(t, "POST", req.Method)

headers = ConvertHeadersToString(req.Header)
headers = ConvertHeadersToBase64String(req.Header)
assert.NotNil(t, headers)

requestString = "GET /_next/static/css/5ecbfdce0834333a.css HTTP/1.1\r\nHost: test.nudgebee.pollux.in\r\nX-Request-ID: 521e623dfebcd7d7a102f151a1120be7\r\nX-Real-IP: 172.31.82.142\r\nX-Forwarded-For: 172.31.82.142\r\nX-Forwarded-Host: test.nudgebee.pollux.in\r\nX-Forwarded-Port: 443\r\nX-Forwarded-Proto: https\r\nX-Forwarded-Scheme: https\r\nX-Scheme: https\r\nsec-ch-ua: \"Google Chrome\";v=\"123\", \"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"123\"\r\nsec-ch-ua-mobile: ?0\r\nuser-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36\r\nsec-ch-ua-platform: \"macOS\"\r\naccept: */*\r\nsec-fetch-site: same-origin\r\nsec-fetch-mode: cors\r\nsec-fetch-dest: empty\r\nreferer: https://test.nudgebee.pollux.in/home?accountId=6048794f-73c7-430a-a308-4d76f1b3b1e8\r\naccept-encoding: gzip, deflate, br, zstd\r\naccept-language: en-US,en;q=0.9\r\ncookie: cw_conversation=eyJhbGciOiJIUzI1NiJ9.eyJzb3VyY2VfaWQiOiJjZGZmNDkwMi0zNTZlLTRhMzYtOWUwNS0yYzcwNjNjNzA3MGIiLCJpbmJveF9pZCI6MzMyMjJ9.yDWSO0Ky_OIEAbWTqlHVG1SZtvejdTLdC4dVATz3FQA;\x00g*A\n"
Expand All @@ -168,7 +168,7 @@ func TestParseHost(t *testing.T) {
assert.Equal(t, "test.nudgebee.pollux.in", req.Host)
assert.Equal(t, "GET", req.Method)

headers = ConvertHeadersToString(req.Header)
headers = ConvertHeadersToBase64String(req.Header)
assert.NotNil(t, headers)
}

Expand Down
1 change: 1 addition & 0 deletions flags/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ var (
SensitiveHeader = kingpin.Flag("sensitive-headers", "sanitize headers using patterns").Default("Authorization, Cookie, X-Action-Token").Envar("SENSITIVE_HEADERS").String()
DisableKubeProbe = kingpin.Flag("disable-kube-probe", "disable kube probe trace").Default("true").Envar("DISABLE_KUBE_PROBE").Bool()
DisableSensitiveLogParsing = kingpin.Flag("disable-sensitive-log-parsing", "disable sensitive log parsing").Default("false").Envar("DISABLE_SENSITIVE_LOG_PARSING").Bool()
TraceIdHeaders = kingpin.Flag("trace-id-headers", "trace id headers").Default("Traceparent,X-Request-Id").Envar("TRACE_ID_HEADERS").String()
)

func GetString(fl *string) string {
Expand Down
70 changes: 56 additions & 14 deletions tracing/tracing.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"crypto/tls"
"fmt"
"net/http"
"strings"
"time"
"unicode/utf8"

Expand Down Expand Up @@ -152,13 +154,26 @@ type Trace struct {
commonAttrs []attribute.KeyValue
}

func (t *Trace) createSpan(name string, duration time.Duration, error bool, attrs ...attribute.KeyValue) {
func (t *Trace) createSpan(name string, duration time.Duration, error bool, traceId string, attrs ...attribute.KeyValue) {
if t.tracer.otel == nil {
return
}
end := time.Now()
start := end.Add(-duration)
_, span := t.tracer.otel.Start(nil, name, trace.WithTimestamp(start), trace.WithSpanKind(trace.SpanKindClient))
ctx := context.Background()
if traceId != "" {
traceId = ParseTraceIdHeaders(traceId)
traceID, err := trace.TraceIDFromHex(traceId)
if err != nil {
context.Background()
}
spanContext := trace.NewSpanContext(trace.SpanContextConfig{
TraceID: traceID,
})
ctx = trace.ContextWithRemoteSpanContext(ctx, spanContext)
}

_, span := t.tracer.otel.Start(ctx, name, trace.WithTimestamp(start), trace.WithSpanKind(trace.SpanKindClient))
span.SetAttributes(attrs...)
span.SetAttributes(t.commonAttrs...)
if error {
Expand All @@ -167,7 +182,20 @@ func (t *Trace) createSpan(name string, duration time.Duration, error bool, attr
span.End(trace.WithTimestamp(end))
}

func (t *Trace) HttpRequest(method, path string, status l7.Status, duration time.Duration, requestSize uint64, payload string, headers string, response string, host string) {
func ParseTraceIdHeaders(traceId string) string {
if traceId == "" {
return ""
}
// if its a traceparent header, extract the traceId
parts := strings.Split(traceId, "-")
if len(parts) == 4 {
traceId = parts[1]
}

return traceId
}

func (t *Trace) HttpRequest(method, path string, status l7.Status, duration time.Duration, requestSize uint64, payload string, headers http.Header, response string, host string) {
if t == nil || method == "" {
return
}
Expand All @@ -179,9 +207,6 @@ func (t *Trace) HttpRequest(method, path string, status l7.Status, duration time
requestPayload = payload
}
requestHeaders := ""
if utf8.ValidString(headers) {
requestHeaders = headers
}
responsePayload := ""
if utf8.ValidString(response) {
responsePayload = response
Expand All @@ -195,7 +220,24 @@ func (t *Trace) HttpRequest(method, path string, status l7.Status, duration time
requestHost = host
}

traceId := ""
if headers != nil {
requestHeaders = l7.ConvertHeadersToBase64String(headers)
traceIdHeaders := strings.Split(*flags.TraceIdHeaders, ",")
for _, header := range traceIdHeaders {
if id := headers.Get(header); id != "" {
// check for lowercase header
traceId = id
break
} else if id := headers.Get(strings.ToLower(header)); id != "" {
traceId = id
break
}
}
}

t.createSpan(method, duration, status >= 400,
traceId,
semconv.HTTPURL(fmt.Sprintf("http://%s%s", requestHost, requestPath)),
semconv.HTTPMethod(method),
semconv.HTTPStatusCode(int(status)),
Expand Down Expand Up @@ -225,7 +267,7 @@ func (t *Trace) Http2Request(method, path, scheme string, status l7.Status, dura
requestPayload = payload
}

t.createSpan(method, duration, status > 400,
t.createSpan(method, duration, status > 400, "",
semconv.HTTPURL(fmt.Sprintf("%s://%s%s", scheme, t.destination.String(), path)),
semconv.HTTPMethod(method),
semconv.HTTPStatusCode(int(status)),
Expand All @@ -237,7 +279,7 @@ func (t *Trace) PostgresQuery(query string, error bool, duration time.Duration)
if t == nil || query == "" {
return
}
t.createSpan("query", duration, error,
t.createSpan("query", duration, error, "",
semconv.DBSystemPostgreSQL,
semconv.DBStatement(query),
)
Expand All @@ -247,7 +289,7 @@ func (t *Trace) MysqlQuery(query string, error bool, duration time.Duration) {
if t == nil || query == "" {
return
}
t.createSpan("query", duration, error,
t.createSpan("query", duration, error, "",
semconv.DBSystemMySQL,
semconv.DBStatement(query),
)
Expand All @@ -257,7 +299,7 @@ func (t *Trace) MongoQuery(query string, error bool, duration time.Duration) {
if t == nil || query == "" {
return
}
t.createSpan("query", duration, error,
t.createSpan("query", duration, error, "",
semconv.DBSystemMongoDB,
semconv.DBStatement(query),
)
Expand All @@ -276,7 +318,7 @@ func (t *Trace) MemcachedQuery(cmd string, items []string, error bool, duration
} else if len(items) > 1 {
attrs = append(attrs, MemcacheDBItemKeyName.StringSlice(items))
}
t.createSpan(cmd, duration, error, attrs...)
t.createSpan(cmd, duration, error, "", attrs...)
}

func (t *Trace) RedisQuery(cmd, args string, error bool, duration time.Duration) {
Expand All @@ -287,7 +329,7 @@ func (t *Trace) RedisQuery(cmd, args string, error bool, duration time.Duration)
if args != "" {
statement += " " + args
}
t.createSpan(cmd, duration, error,
t.createSpan(cmd, duration, error, "",
semconv.DBSystemRedis,
semconv.DBOperation(cmd),
semconv.DBStatement(statement),
Expand All @@ -298,7 +340,7 @@ func (t *Trace) ClickhouseQuery(query string, error bool, duration time.Duration
if t == nil {
return
}
t.createSpan("query", duration, error,
t.createSpan("query", duration, error, "",
semconv.DBSystemClickhouse,
semconv.DBStatement(query),
)
Expand All @@ -315,7 +357,7 @@ func (t *Trace) ZookeeperRequest(op string, args string, status l7.Status, durat
if args != "" {
statement += " " + args
}
t.createSpan(op, duration, status.Zookeeper() != "ok",
t.createSpan(op, duration, status.Zookeeper() != "ok", "",
semconv.DBSystemKey.String("zookeeper"),
semconv.DBOperation(op),
semconv.DBStatementKey.String(statement),
Expand Down