From 1c9b35d17a3f7c8acc8bd88e04f5334f02edcd44 Mon Sep 17 00:00:00 2001 From: Rudrakh Panigrahi Date: Thu, 4 Apr 2024 17:12:26 +0530 Subject: [PATCH] add http.send request attribute to ignore headers for caching key Signed-off-by: Rudrakh Panigrahi --- topdown/http.go | 39 +++++++++++++++++++++++++++++++++++++-- topdown/http_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/topdown/http.go b/topdown/http.go index 22e6843d4f..d68ef3d383 100644 --- a/topdown/http.go +++ b/topdown/http.go @@ -68,6 +68,7 @@ var allowedKeyNames = [...]string{ "raise_error", "caching_mode", "max_retry_attempts", + "cache_ignored_headers", } // ref: https://www.rfc-editor.org/rfc/rfc7231#section-6.1 @@ -168,7 +169,11 @@ func getHTTPResponse(bctx BuiltinContext, req ast.Object) (*ast.Term, error) { bctx.Metrics.Timer(httpSendLatencyMetricKey).Start() - reqExecutor, err := newHTTPRequestExecutor(bctx, req) + key, err := getKeyFromRequest(req) + if err != nil { + return nil, err + } + reqExecutor, err := newHTTPRequestExecutor(bctx, key) if err != nil { return nil, err } @@ -198,6 +203,36 @@ func getHTTPResponse(bctx BuiltinContext, req ast.Object) (*ast.Term, error) { return ast.NewTerm(resp), nil } +// getKeyFromRequest returns a key to be used for caching HTTP responses +// deletes headers from request object mentioned in cache_ignored_headers +func getKeyFromRequest(req ast.Object) (ast.Object, error) { + var cacheIgnoredHeaders []string + var allHeaders map[string]interface{} + cacheIgnoredHeadersTerm := req.Get(ast.StringTerm("cache_ignored_headers")) + allHeadersTerm := req.Get(ast.StringTerm("headers")) + if cacheIgnoredHeadersTerm != nil && allHeadersTerm != nil { + err := ast.As(cacheIgnoredHeadersTerm.Value, &cacheIgnoredHeaders) + if err != nil { + return nil, err + } + err = ast.As(allHeadersTerm.Value, &allHeaders) + if err != nil { + return nil, err + } + for _, header := range cacheIgnoredHeaders { + delete(allHeaders, header) + } + val, err := ast.InterfaceToValue(allHeaders) + if err != nil { + return nil, err + } + allHeadersTerm.Value = val + req.Insert(ast.StringTerm("headers"), allHeadersTerm) + } + req.Insert(ast.StringTerm("cache_ignored_headers"), ast.NullTerm()) + return req, nil +} + func init() { createAllowedKeys() createCacheableHTTPStatusCodes() @@ -482,7 +517,7 @@ func createHTTPRequest(bctx BuiltinContext, obj ast.Object) (*http.Request, *htt case "cache", "caching_mode", "force_cache", "force_cache_duration_seconds", "force_json_decode", "force_yaml_decode", - "raise_error", "max_retry_attempts": // no-op + "raise_error", "max_retry_attempts", "cache_ignored_headers": // no-op default: return nil, nil, fmt.Errorf("invalid parameter %q", key) } diff --git a/topdown/http_test.go b/topdown/http_test.go index 6f35c3aa08..2b5b9b4903 100644 --- a/topdown/http_test.go +++ b/topdown/http_test.go @@ -1012,6 +1012,46 @@ func TestHTTPSendCaching(t *testing.T) { response: `{"x": 1}`, expectedReqCount: 3, }, + { + note: "http.send GET different headers but still cached because ignored", + ruleTemplate: `p = x { + r1 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v2"}, "cache_ignored_headers": ["h2"]}) + r2 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v3"}, "cache_ignored_headers": ["h2"]}) # cached + x = r1.body + }`, + response: `{"x": 1}`, + expectedReqCount: 1, + }, + { + note: "http.send GET cache miss different headers (force_cache enabled)", + ruleTemplate: `p = x { + r1 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v2"}, "force_cache": true, "force_cache_duration_seconds": 300}) + r2 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v3"}, "force_cache": true, "force_cache_duration_seconds": 300}) + x = r1.body + }`, + response: `{"x": 1}`, + expectedReqCount: 2, + }, + { + note: "http.send GET different headers but still cached because ignored (force_cache enabled)", + ruleTemplate: `p = x { + r1 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v2"}, "force_cache": true, "force_cache_duration_seconds": 300, "cache_ignored_headers": ["h2"]}) + r2 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v3"}, "force_cache": true, "force_cache_duration_seconds": 300, "cache_ignored_headers": ["h2"]}) # cached + x = r1.body + }`, + response: `{"x": 1}`, + expectedReqCount: 1, + }, + { + note: "http.send GET different cache_ignored_headers but still cached (force_cache enabled)", + ruleTemplate: `p = x { + r1 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v2"}, "force_cache": true, "force_cache_duration_seconds": 300, "cache_ignored_headers": ["h2"]}) + r2 = http.send({"method": "get", "url": "%URL%", "force_json_decode": true, "headers": {"h1": "v1", "h2": "v2"}, "force_cache": true, "force_cache_duration_seconds": 300, "cache_ignored_headers": ["h2", "h3"]}) # cached + x = r1.body + }`, + response: `{"x": 1}`, + expectedReqCount: 1, + }, { note: "http.send POST cache miss different body", ruleTemplate: `p = x {