From 5103ea7b405419224eb7ecb972bd6c0ffd70d275 Mon Sep 17 00:00:00 2001 From: Robert O Butts Date: Thu, 4 Jun 2020 12:59:19 -0600 Subject: [PATCH] Add Raw Remap Range Directive (#4757) Adds a __RANGE_DIRECTIVE__ to the Raw Remap processing, to insert the Delivery Service Range Request Handling directives there, instead of before the Raw Remap. Normally, Raw Remap always needs to be at the end of the remap line, but in some cases, specifically when custom remaps are modifying the Range header, it's necessary to put the Range Request directives (@plugin=slice.so, @plugin=caceh_raange_requests.so, etc) after the custom Raw Remap text. This makes that possible. (cherry picked from commit fc7a6b928d7e54186dd21b2f807991c07bbe008c) --- CHANGELOG.md | 1 + docs/source/overview/delivery_services.rst | 7 + lib/go-atscfg/remapdotconfig.go | 16 +- lib/go-atscfg/remapdotconfig_test.go | 228 +++++++++++++++++++++ 4 files changed, 249 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d10c7506d3..4a83d82d02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Fixed #4541: traffic_ops_server.js searches based on which= and can find data for multiple servers - Fixed POST deliveryservices/request (designed to simple send an email) regression which erroneously required deep caching type and routing name. [Related github issue](https://github.com/apache/trafficcontrol/issues/4735) - Fixed `maxRevalDurationDays` validation for `POST /api/1.x/user/current/jobs` and added that validation to the `/api/x/jobs` endpoints +- Added Delivery Service Raw Remap `__RANGE_DIRECTIVE__` directive to allow inserting the Range Directive after the Raw Remap text. This allows Raw Remaps which manipulate the Range. ### Deprecated/Removed - The Traffic Ops `db/admin.pl` script has now been removed. Please use the `db/admin` binary instead. diff --git a/docs/source/overview/delivery_services.rst b/docs/source/overview/delivery_services.rst index 7d5d2174af..353b2eaa9d 100644 --- a/docs/source/overview/delivery_services.rst +++ b/docs/source/overview/delivery_services.rst @@ -643,6 +643,13 @@ For HTTP and DNS-:ref:`Routed ` Delivery Services, this will be added | remapText | In Traffic Ops source code and :ref:`to-api` requests/responses | unchanged (``text``, ``string`` etc.) | +-----------+-----------------------------------------------------------------+---------------------------------------+ +Directives +""" + +The Raw Remap text is ordinarily added at the end of the line, after everything else. However, it may be necessary to add Range Request Handling after the Raw Remap. For example, if you have a plugin which manipulates the Range header. In this case, you can insert the text ``__RANGE_DIRECTIVE__`` in the Raw Remap text, and the range request handling directives will be added at that point. + +For example, if you have an Apache Traffic Server lua plugin which manipulates the range, and are using Slice Range Request Handling which needs to run after your plugin, you can set a Raw Remap, ``@plugin=tslua.so @pparam=range.lua __RANGE_DIRECTIVE__``, and the ``@plugin=slice.so`` range directive will be inserted after your plugin. + .. _ds-regex-remap: Regex Remap Expression diff --git a/lib/go-atscfg/remapdotconfig.go b/lib/go-atscfg/remapdotconfig.go index 7b916868e9..ad9c1bf0d4 100644 --- a/lib/go-atscfg/remapdotconfig.go +++ b/lib/go-atscfg/remapdotconfig.go @@ -206,6 +206,8 @@ func GetServerConfigRemapDotConfigForEdge( return text } +const RemapConfigRangeDirective = `__RANGE_DIRECTIVE__` + // BuildRemapLine builds the remap line for the given server and delivery service. // The cacheKeyConfigParams map may be nil, if this ds profile had no cache key config params. func BuildRemapLine(cacheURLConfigParams map[string]string, atsMajorVersion int, server *ServerInfo, pData map[string]string, text string, ds RemapConfigDSData, mapFrom string, mapTo string, cacheKeyConfigParams map[string]string) string { @@ -284,15 +286,23 @@ func BuildRemapLine(cacheURLConfigParams map[string]string, atsMajorVersion int, if ds.RegexRemap != nil && *ds.RegexRemap != "" { text += ` @plugin=regex_remap.so @pparam=regex_remap_` + ds.Name + ".config" } + + rangeReqTxt := "" if ds.RangeRequestHandling != nil { if *ds.RangeRequestHandling == tc.RangeRequestHandlingBackgroundFetch { - text += ` @plugin=background_fetch.so @pparam=bg_fetch.config` + rangeReqTxt = ` @plugin=background_fetch.so @pparam=bg_fetch.config` } else if *ds.RangeRequestHandling == tc.RangeRequestHandlingCacheRangeRequest { - text += ` @plugin=cache_range_requests.so ` + rangeReqTxt = ` @plugin=cache_range_requests.so ` } else if *ds.RangeRequestHandling == tc.RangeRequestHandlingSlice && ds.RangeSliceBlockSize != nil { - text += ` @plugin=slice.so @pparam=--blockbytes=` + strconv.Itoa(*ds.RangeSliceBlockSize) + ` @plugin=cache_range_requests.so ` + rangeReqTxt = ` @plugin=slice.so @pparam=--blockbytes=` + strconv.Itoa(*ds.RangeSliceBlockSize) + ` @plugin=cache_range_requests.so ` } } + if ds.RemapText != nil && *ds.RemapText != "" && strings.Contains(*ds.RemapText, RemapConfigRangeDirective) { + *ds.RemapText = strings.Replace(*ds.RemapText, `__RANGE_DIRECTIVE__`, rangeReqTxt, 1) + } else { + text += rangeReqTxt + } + if ds.RemapText != nil && *ds.RemapText != "" { text += " " + *ds.RemapText } diff --git a/lib/go-atscfg/remapdotconfig_test.go b/lib/go-atscfg/remapdotconfig_test.go index 52f9b2e1a2..2030b4301a 100644 --- a/lib/go-atscfg/remapdotconfig_test.go +++ b/lib/go-atscfg/remapdotconfig_test.go @@ -4872,6 +4872,234 @@ func TestMakeRemapDotConfigEdgeRangeRequestSlice(t *testing.T) { t.Errorf("expected remap on edge server with ds slice range request handling to contain block size for the slice plugin, actual '%v'", txt) } } + +func TestMakeRemapDotConfigRawRemapRangeDirective(t *testing.T) { + serverName := tc.CacheName("server0") + toToolName := "to0" + toURL := "trafficops.example.net" + atsMajorVersion := 7 + + cacheURLConfigParams := map[string]string{ + "location": "notinconfig", + } + + dsProfilesCacheKeyConfigParams := map[int]map[string]string{ + 49: map[string]string{ + "cachekeykey": "cachekeyval", + }, + 44: map[string]string{ + "shouldnotincludeotherprofile": "shouldnotincludeotherprofileval", + }, + } + + serverPackageParamData := map[string]string{ + "dscp_remap_no": "notused", + } + + serverInfo := &ServerInfo{ + CacheGroupID: 42, + CDN: "mycdn", + CDNID: 43, + DomainName: "mydomain", + HostName: "myhost", + HTTPSPort: 12443, + ID: 44, + IP: "192.168.2.4", + ParentCacheGroupID: 45, + ParentCacheGroupType: "CGType4", + ProfileID: 46, + ProfileName: "MyProfile", + Port: 12080, + SecondaryParentCacheGroupID: 47, + SecondaryParentCacheGroupType: "MySecondaryParentCG", + Type: "EDGE", + } + + remapDSData := []RemapConfigDSData{ + RemapConfigDSData{ + ID: 48, + Type: "HTTP_LIVE_NATNL", + OriginFQDN: util.StrPtr("myorigin"), + MidHeaderRewrite: util.StrPtr("mymidrewrite"), + CacheURL: util.StrPtr(""), + RangeRequestHandling: util.IntPtr(tc.RangeRequestHandlingSlice), + CacheKeyConfigParams: map[string]string{"cachekeyparamname": "cachekeyparamval"}, + RemapText: util.StrPtr("@plugin=tslua.so @pparam=my-range-manipulator.lua __RANGE_DIRECTIVE__"), + EdgeHeaderRewrite: nil, + SigningAlgorithm: util.StrPtr("foo"), + Name: "mydsname", + QStringIgnore: util.IntPtr(int(tc.QueryStringIgnoreIgnoreInCacheKeyAndPassUp)), + RegexRemap: util.StrPtr(""), + FQPacingRate: util.IntPtr(0), + DSCP: 0, + RoutingName: util.StrPtr("myroutingname"), + MultiSiteOrigin: util.StrPtr("mymso"), + Pattern: util.StrPtr(`mypattern`), + RegexType: util.StrPtr(string(tc.DSMatchTypeHostRegex)), + Domain: util.StrPtr("mydomain"), + RegexSetNumber: util.StrPtr("myregexsetnum"), + OriginShield: util.StrPtr("myoriginshield"), + ProfileID: util.IntPtr(49), + Protocol: util.IntPtr(int(tc.DSProtocolHTTPToHTTPS)), + AnonymousBlockingEnabled: util.BoolPtr(false), + Active: true, + RangeSliceBlockSize: util.IntPtr(262144), + }, + } + + txt := MakeRemapDotConfig(serverName, toToolName, toURL, atsMajorVersion, cacheURLConfigParams, dsProfilesCacheKeyConfigParams, serverPackageParamData, serverInfo, remapDSData) + + txt = strings.TrimSpace(txt) + + testComment(t, txt, string(serverName), toToolName, toURL) + + txtLines := strings.Split(txt, "\n") + + if len(txtLines) != 2 { + t.Fatalf("expected 1 remaps from HTTP_TO_HTTPS DS, actual: '%v' count %v", txt, len(txtLines)) + } + + remapLine := txtLines[1] + + if !strings.HasPrefix(remapLine, "map") { + t.Errorf("expected to start with 'map', actual '%v'", txt) + } + + if !strings.Contains(remapLine, "slice.so") { + t.Errorf("expected remap on edge server with ds slice range request handling to contain background fetch plugin, actual '%v'", txt) + } + + if !strings.Contains(remapLine, "cache_range_requests.so") { + t.Errorf("expected remap on edge server with ds slice range request handling to contain cache_range_requests plugin, actual '%v'", txt) + } + + if !strings.Contains(remapLine, "pparam=--blockbytes=262144") { + t.Errorf("expected remap on edge server with ds slice range request handling to contain block size for the slice plugin, actual '%v'", txt) + } + + if !strings.Contains(remapLine, "@plugin=tslua.so @pparam=my-range-manipulator.lua @plugin=slice.so @pparam=--blockbytes=262144 @plugin=cache_range_requests.so") { + t.Errorf("expected raw remap to come after range directive, actual '%v'", txt) + } + if strings.Contains(remapLine, "__RANGE_DIRECTIVE__") { + t.Errorf("expected raw remap range directive to be replaced, actual '%v'", txt) + } + if strings.Count(remapLine, "slice.so") != 1 { + t.Errorf("expected raw remap range directive to be replaced not duplicated, actual '%v'", txt) + } +} + +func TestMakeRemapDotConfigRawRemapWithoutRangeDirective(t *testing.T) { + serverName := tc.CacheName("server0") + toToolName := "to0" + toURL := "trafficops.example.net" + atsMajorVersion := 7 + + cacheURLConfigParams := map[string]string{ + "location": "notinconfig", + } + + dsProfilesCacheKeyConfigParams := map[int]map[string]string{ + 49: map[string]string{ + "cachekeykey": "cachekeyval", + }, + 44: map[string]string{ + "shouldnotincludeotherprofile": "shouldnotincludeotherprofileval", + }, + } + + serverPackageParamData := map[string]string{ + "dscp_remap_no": "notused", + } + + serverInfo := &ServerInfo{ + CacheGroupID: 42, + CDN: "mycdn", + CDNID: 43, + DomainName: "mydomain", + HostName: "myhost", + HTTPSPort: 12443, + ID: 44, + IP: "192.168.2.4", + ParentCacheGroupID: 45, + ParentCacheGroupType: "CGType4", + ProfileID: 46, + ProfileName: "MyProfile", + Port: 12080, + SecondaryParentCacheGroupID: 47, + SecondaryParentCacheGroupType: "MySecondaryParentCG", + Type: "EDGE", + } + + remapDSData := []RemapConfigDSData{ + RemapConfigDSData{ + ID: 48, + Type: "HTTP_LIVE_NATNL", + OriginFQDN: util.StrPtr("myorigin"), + MidHeaderRewrite: util.StrPtr("mymidrewrite"), + CacheURL: util.StrPtr(""), + RangeRequestHandling: util.IntPtr(tc.RangeRequestHandlingSlice), + CacheKeyConfigParams: map[string]string{"cachekeyparamname": "cachekeyparamval"}, + RemapText: util.StrPtr("@plugin=tslua.so @pparam=my-range-manipulator.lua"), + EdgeHeaderRewrite: nil, + SigningAlgorithm: util.StrPtr("foo"), + Name: "mydsname", + QStringIgnore: util.IntPtr(int(tc.QueryStringIgnoreIgnoreInCacheKeyAndPassUp)), + RegexRemap: util.StrPtr(""), + FQPacingRate: util.IntPtr(0), + DSCP: 0, + RoutingName: util.StrPtr("myroutingname"), + MultiSiteOrigin: util.StrPtr("mymso"), + Pattern: util.StrPtr(`mypattern`), + RegexType: util.StrPtr(string(tc.DSMatchTypeHostRegex)), + Domain: util.StrPtr("mydomain"), + RegexSetNumber: util.StrPtr("myregexsetnum"), + OriginShield: util.StrPtr("myoriginshield"), + ProfileID: util.IntPtr(49), + Protocol: util.IntPtr(int(tc.DSProtocolHTTPToHTTPS)), + AnonymousBlockingEnabled: util.BoolPtr(false), + Active: true, + RangeSliceBlockSize: util.IntPtr(262144), + }, + } + + txt := MakeRemapDotConfig(serverName, toToolName, toURL, atsMajorVersion, cacheURLConfigParams, dsProfilesCacheKeyConfigParams, serverPackageParamData, serverInfo, remapDSData) + + txt = strings.TrimSpace(txt) + + testComment(t, txt, string(serverName), toToolName, toURL) + + txtLines := strings.Split(txt, "\n") + + if len(txtLines) != 2 { + t.Fatalf("expected 1 remaps from HTTP_TO_HTTPS DS, actual: '%v' count %v", txt, len(txtLines)) + } + + remapLine := txtLines[1] + + if !strings.HasPrefix(remapLine, "map") { + t.Errorf("expected to start with 'map', actual '%v'", txt) + } + + if !strings.Contains(remapLine, "slice.so") { + t.Errorf("expected remap on edge server with ds slice range request handling to contain background fetch plugin, actual '%v'", txt) + } + + if !strings.Contains(remapLine, "cache_range_requests.so") { + t.Errorf("expected remap on edge server with ds slice range request handling to contain cache_range_requests plugin, actual '%v'", txt) + } + + if !strings.Contains(remapLine, "pparam=--blockbytes=262144") { + t.Errorf("expected remap on edge server with ds slice range request handling to contain block size for the slice plugin, actual '%v'", txt) + } + + if !strings.HasSuffix(remapLine, "@plugin=tslua.so @pparam=my-range-manipulator.lua") { + t.Errorf("expected raw remap without range directive at end of remap line, actual '%v'", txt) + } + if strings.Count(remapLine, "slice.so") != 1 { + t.Errorf("expected raw remap range directive to not be duplicated, actual '%v'", txt) + } +} + func TestMakeRemapDotConfigEdgeRangeRequestCache(t *testing.T) { serverName := tc.CacheName("server0") toToolName := "to0"