Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

x-pack/filebeat/input/httpjson: improve template evaluation logging #36668

Merged
merged 1 commit into from
Sep 28, 2023
Merged

x-pack/filebeat/input/httpjson: improve template evaluation logging #36668

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
- Disable warning message about ingest pipeline loading when running under Elastic Agent. {pull}36659[36659]
- Add input metrics to http_endpoint input. {issue}36402[36402] {pull}36427[36427]
- Update mito CEL extension library to v1.6.0. {pull}36651[36651]
- Improve template evaluation logging for HTTPJSON input. {pull}36668[36668]

*Auditbeat*

Expand Down
2 changes: 1 addition & 1 deletion x-pack/filebeat/input/httpjson/cursor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func (c *cursor) update(trCtx *transformContext) {
}

for k, cfg := range c.cfg {
v, _ := cfg.Value.Execute(trCtx, transformable{}, "", cfg.Default, c.log)
v, _ := cfg.Value.Execute(trCtx, transformable{}, k, cfg.Default, c.log)
if v != "" || !cfg.mustIgnoreEmptyValue() {
_, _ = c.state.Put(k, v)
c.log.Debugf("cursor.%s stored with %s", k, v)
Expand Down
2 changes: 2 additions & 0 deletions x-pack/filebeat/input/httpjson/input_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1173,9 +1173,9 @@
</item>
</order>
`
io.ReadAll(r.Body)

Check failure on line 1176 in x-pack/filebeat/input/httpjson/input_test.go

View workflow job for this annotation

GitHub Actions / lint (linux) 

Error return value of `io.ReadAll` is not checked (errcheck)
r.Body.Close()
w.Write([]byte(text))

Check failure on line 1178 in x-pack/filebeat/input/httpjson/input_test.go

View workflow job for this annotation

GitHub Actions / lint (linux) 

Error return value of `w.Write` is not checked (errcheck)
})
server := httptest.NewServer(r)
config["request.url"] = server.URL
Expand Down Expand Up @@ -1248,6 +1248,8 @@
}

func TestInput(t *testing.T) {
logp.TestingSetup()

Check failure on line 1251 in x-pack/filebeat/input/httpjson/input_test.go

View workflow job for this annotation

GitHub Actions / lint (linux) 

Error return value of `logp.TestingSetup` is not checked (errcheck)

for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
if test.skipReason != "" {
Expand Down
6 changes: 3 additions & 3 deletions x-pack/filebeat/input/httpjson/rate_limiter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) {
ctx := emptyTransformContext()
ctx.updateLastResponse(response{header: resp.Header.Clone()})

remaining, _ := r.remaining.Execute(ctx, tr, "", nil, r.log)
remaining, _ := r.remaining.Execute(ctx, tr, "rate-limit_remaining", nil, r.log)
if remaining == "" {
return 0, errors.New("remaining value is empty")
}
Expand All @@ -119,7 +119,7 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) {
if r.earlyLimit != nil {
earlyLimit := *r.earlyLimit
if earlyLimit > 0 && earlyLimit < 1 {
limit, _ := r.limit.Execute(ctx, tr, "", nil, r.log)
limit, _ := r.limit.Execute(ctx, tr, "early_limit", nil, r.log)
if limit != "" {
l, err := strconv.ParseInt(limit, 10, 64)
if err == nil {
Expand All @@ -141,7 +141,7 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) {
return 0, nil
}

reset, _ := r.reset.Execute(ctx, tr, "", nil, r.log)
reset, _ := r.reset.Execute(ctx, tr, "rate-limit_reset", nil, r.log)
if reset == "" {
return 0, errors.New("reset value is empty")
}
Expand Down
2 changes: 1 addition & 1 deletion x-pack/filebeat/input/httpjson/request.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ func evaluateResponse(expression *valueTpl, data []byte, log *logp.Logger) (bool
lastResponse: &response{body: dataMap},
}

val, err := expression.Execute(paramCtx, tr, "", nil, log)
val, err := expression.Execute(paramCtx, tr, "response_evaluation", nil, log)
if err != nil {
return false, fmt.Errorf("error while evaluating expression: %w", err)
}
Expand Down
6 changes: 3 additions & 3 deletions x-pack/filebeat/input/httpjson/value_tpl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func (t *valueTpl) Unpack(in string) error {
func (t *valueTpl) Execute(trCtx *transformContext, tr transformable, targetName string, defaultVal *valueTpl, log *logp.Logger) (val string, err error) {
fallback := func(err error) (string, error) {
if defaultVal != nil {
log.Debugf("template execution: falling back to default value")
log.Debugw("template execution: falling back to default value", "target", targetName)
return defaultVal.Execute(emptyTransformContext(), transformable{}, targetName, nil, log)
}
return "", err
Expand All @@ -107,7 +107,7 @@ func (t *valueTpl) Execute(trCtx *transformContext, tr transformable, targetName
val, err = fallback(errExecutingTemplate)
}
if err != nil {
log.Debugf("template execution failed: %v", err)
log.Debugw("template execution failed", "target", targetName, "error", err)
}
tryDebugTemplateValue(targetName, val, log)
}()
Expand Down Expand Up @@ -142,7 +142,7 @@ func tryDebugTemplateValue(target, val string, log *logp.Logger) {
case "Authorization", "Proxy-Authorization":
// ignore filtered headers
default:
log.Debugf("template execution: evaluated template %q", val)
log.Debugw("evaluated template", "target", target, "value", val)
}
}

Expand Down
4 changes: 3 additions & 1 deletion x-pack/filebeat/input/httpjson/value_tpl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
)

func TestValueTpl(t *testing.T) {
logp.TestingSetup()

Check failure on line 22 in x-pack/filebeat/input/httpjson/value_tpl_test.go

View workflow job for this annotation

GitHub Actions / lint (linux) 

Error return value of `logp.TestingSetup` is not checked (errcheck)

cases := []struct {
name string
value string
Expand Down Expand Up @@ -764,7 +766,7 @@
assert.NoError(t, defTpl.Unpack(tc.paramDefVal))
}

got, err := tpl.Execute(tc.paramCtx, tc.paramTr, "", defTpl, logp.NewLogger(""))
got, err := tpl.Execute(tc.paramCtx, tc.paramTr, tc.name, defTpl, logp.NewLogger(""))
assert.Equal(t, tc.expectedVal, got)
if tc.expectedError == "" {
assert.NoError(t, err)
Expand Down