[BACKPORT 4.1.x] Add Raw Remap Range Directive #4757

CHANGELOG.md

@@ -65,6 +65,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Fixed #4541: traffic_ops_server.js searches based on which= and can find data for multiple servers
 - Fixed POST deliveryservices/request (designed to simple send an email) regression which erroneously required deep caching type and routing name. [Related github issue](https://github.com/apache/trafficcontrol/issues/4735)
 - Fixed `maxRevalDurationDays` validation for `POST /api/1.x/user/current/jobs` and added that validation to the `/api/x/jobs` endpoints
+- Added Delivery Service Raw Remap `__RANGE_DIRECTIVE__` directive to allow inserting the Range Directive after the Raw Remap text. This allows Raw Remaps which manipulate the Range.
 
 ### Deprecated/Removed
 - The Traffic Ops `db/admin.pl` script has now been removed. Please use the `db/admin` binary instead.

docs/source/overview/delivery_services.rst

@@ -643,6 +643,13 @@ For HTTP and DNS-:ref:`Routed <ds-types>` Delivery Services, this will be added
 	| remapText | In Traffic Ops source code and :ref:`to-api` requests/responses | unchanged (``text``, ``string`` etc.) |
 	+-----------+-----------------------------------------------------------------+---------------------------------------+
 
+Directives
+"""
+
+The Raw Remap text is ordinarily added at the end of the line, after everything else. However, it may be necessary to add Range Request Handling after the Raw Remap. For example, if you have a plugin which manipulates the Range header. In this case, you can insert the text ``__RANGE_DIRECTIVE__`` in the Raw Remap text, and the range request handling directives will be added at that point.
+
+For example, if you have an Apache Traffic Server lua plugin which manipulates the range, and are using Slice Range Request Handling which needs to run after your plugin, you can set a Raw Remap, ``@plugin=tslua.so @pparam=range.lua __RANGE_DIRECTIVE__``, and the ``@plugin=slice.so`` range directive will be inserted after your plugin.
+
 .. _ds-regex-remap:
 
 Regex Remap Expression

lib/go-atscfg/remapdotconfig.go

@@ -206,6 +206,8 @@ func GetServerConfigRemapDotConfigForEdge(
 	return text
 }
 
+const RemapConfigRangeDirective = `__RANGE_DIRECTIVE__`
+
 // BuildRemapLine builds the remap line for the given server and delivery service.
 // The cacheKeyConfigParams map may be nil, if this ds profile had no cache key config params.
 func BuildRemapLine(cacheURLConfigParams map[string]string, atsMajorVersion int, server *ServerInfo, pData map[string]string, text string, ds RemapConfigDSData, mapFrom string, mapTo string, cacheKeyConfigParams map[string]string) string {
@@ -284,15 +286,23 @@ func BuildRemapLine(cacheURLConfigParams map[string]string, atsMajorVersion int,
 	if ds.RegexRemap != nil && *ds.RegexRemap != "" {
 		text += ` @plugin=regex_remap.so @pparam=regex_remap_` + ds.Name + ".config"
 	}
+
+	rangeReqTxt := ""
 	if ds.RangeRequestHandling != nil {
 		if *ds.RangeRequestHandling == tc.RangeRequestHandlingBackgroundFetch {
-			text += ` @plugin=background_fetch.so @pparam=bg_fetch.config`
+			rangeReqTxt = ` @plugin=background_fetch.so @pparam=bg_fetch.config`
 		} else if *ds.RangeRequestHandling == tc.RangeRequestHandlingCacheRangeRequest {
-			text += ` @plugin=cache_range_requests.so `
+			rangeReqTxt = ` @plugin=cache_range_requests.so `
 		} else if *ds.RangeRequestHandling == tc.RangeRequestHandlingSlice && ds.RangeSliceBlockSize != nil {
-			text += ` @plugin=slice.so @pparam=--blockbytes=` + strconv.Itoa(*ds.RangeSliceBlockSize) + ` @plugin=cache_range_requests.so	`
+			rangeReqTxt = ` @plugin=slice.so @pparam=--blockbytes=` + strconv.Itoa(*ds.RangeSliceBlockSize) + ` @plugin=cache_range_requests.so	`
 		}
 	}
+	if ds.RemapText != nil && *ds.RemapText != "" && strings.Contains(*ds.RemapText, RemapConfigRangeDirective) {
+		*ds.RemapText = strings.Replace(*ds.RemapText, `__RANGE_DIRECTIVE__`, rangeReqTxt, 1)
+	} else {
+		text += rangeReqTxt
+	}
+
 	if ds.RemapText != nil && *ds.RemapText != "" {
 		text += " " + *ds.RemapText
 	}

lib/go-atscfg/remapdotconfig_test.go

@@ -4872,6 +4872,234 @@ func TestMakeRemapDotConfigEdgeRangeRequestSlice(t *testing.T) {
 		t.Errorf("expected remap on edge server with ds slice range request handling to contain block size for the slice plugin, actual '%v'", txt)
 	}
 }
+
+func TestMakeRemapDotConfigRawRemapRangeDirective(t *testing.T) {
+	serverName := tc.CacheName("server0")
+	toToolName := "to0"
+	toURL := "trafficops.example.net"
+	atsMajorVersion := 7
+
+	cacheURLConfigParams := map[string]string{
+		"location": "notinconfig",
+	}
+
+	dsProfilesCacheKeyConfigParams := map[int]map[string]string{
+		49: map[string]string{
+			"cachekeykey": "cachekeyval",
+		},
+		44: map[string]string{
+			"shouldnotincludeotherprofile": "shouldnotincludeotherprofileval",
+		},
+	}
+
+	serverPackageParamData := map[string]string{
+		"dscp_remap_no": "notused",
+	}
+
+	serverInfo := &ServerInfo{
+		CacheGroupID:                  42,
+		CDN:                           "mycdn",
+		CDNID:                         43,
+		DomainName:                    "mydomain",
+		HostName:                      "myhost",
+		HTTPSPort:                     12443,
+		ID:                            44,
+		IP:                            "192.168.2.4",
+		ParentCacheGroupID:            45,
+		ParentCacheGroupType:          "CGType4",
+		ProfileID:                     46,
+		ProfileName:                   "MyProfile",
+		Port:                          12080,
+		SecondaryParentCacheGroupID:   47,
+		SecondaryParentCacheGroupType: "MySecondaryParentCG",
+		Type:                          "EDGE",
+	}
+
+	remapDSData := []RemapConfigDSData{
+		RemapConfigDSData{
+			ID:                       48,
+			Type:                     "HTTP_LIVE_NATNL",
+			OriginFQDN:               util.StrPtr("myorigin"),
+			MidHeaderRewrite:         util.StrPtr("mymidrewrite"),
+			CacheURL:                 util.StrPtr(""),
+			RangeRequestHandling:     util.IntPtr(tc.RangeRequestHandlingSlice),
+			CacheKeyConfigParams:     map[string]string{"cachekeyparamname": "cachekeyparamval"},
+			RemapText:                util.StrPtr("@plugin=tslua.so @pparam=my-range-manipulator.lua __RANGE_DIRECTIVE__"),
+			EdgeHeaderRewrite:        nil,
+			SigningAlgorithm:         util.StrPtr("foo"),
+			Name:                     "mydsname",
+			QStringIgnore:            util.IntPtr(int(tc.QueryStringIgnoreIgnoreInCacheKeyAndPassUp)),
+			RegexRemap:               util.StrPtr(""),
+			FQPacingRate:             util.IntPtr(0),
+			DSCP:                     0,
+			RoutingName:              util.StrPtr("myroutingname"),
+			MultiSiteOrigin:          util.StrPtr("mymso"),
+			Pattern:                  util.StrPtr(`mypattern`),
+			RegexType:                util.StrPtr(string(tc.DSMatchTypeHostRegex)),
+			Domain:                   util.StrPtr("mydomain"),
+			RegexSetNumber:           util.StrPtr("myregexsetnum"),
+			OriginShield:             util.StrPtr("myoriginshield"),
+			ProfileID:                util.IntPtr(49),
+			Protocol:                 util.IntPtr(int(tc.DSProtocolHTTPToHTTPS)),
+			AnonymousBlockingEnabled: util.BoolPtr(false),
+			Active:                   true,
+			RangeSliceBlockSize:      util.IntPtr(262144),
+		},
+	}
+
+	txt := MakeRemapDotConfig(serverName, toToolName, toURL, atsMajorVersion, cacheURLConfigParams, dsProfilesCacheKeyConfigParams, serverPackageParamData, serverInfo, remapDSData)
+
+	txt = strings.TrimSpace(txt)
+
+	testComment(t, txt, string(serverName), toToolName, toURL)
+
+	txtLines := strings.Split(txt, "\n")
+
+	if len(txtLines) != 2 {
+		t.Fatalf("expected 1 remaps from HTTP_TO_HTTPS DS, actual: '%v' count %v", txt, len(txtLines))
+	}
+
+	remapLine := txtLines[1]
+
+	if !strings.HasPrefix(remapLine, "map") {
+		t.Errorf("expected to start with 'map', actual '%v'", txt)
+	}
+
+	if !strings.Contains(remapLine, "slice.so") {
+		t.Errorf("expected remap on edge server with ds slice range request handling to contain background fetch plugin, actual '%v'", txt)
+	}
+
+	if !strings.Contains(remapLine, "cache_range_requests.so") {
+		t.Errorf("expected remap on edge server with ds slice range request handling to contain cache_range_requests plugin, actual '%v'", txt)
+	}
+
+	if !strings.Contains(remapLine, "pparam=--blockbytes=262144") {
+		t.Errorf("expected remap on edge server with ds slice range request handling to contain block size for the slice plugin, actual '%v'", txt)
+	}
+
+	if !strings.Contains(remapLine, "@plugin=tslua.so @pparam=my-range-manipulator.lua  @plugin=slice.so @pparam=--blockbytes=262144 @plugin=cache_range_requests.so") {
+		t.Errorf("expected raw remap to come after range directive, actual '%v'", txt)
+	}
+	if strings.Contains(remapLine, "__RANGE_DIRECTIVE__") {
+		t.Errorf("expected raw remap range directive to be replaced, actual '%v'", txt)
+	}
+	if strings.Count(remapLine, "slice.so") != 1 {
+		t.Errorf("expected raw remap range directive to be replaced not duplicated, actual '%v'", txt)
+	}
+}
+
+func TestMakeRemapDotConfigRawRemapWithoutRangeDirective(t *testing.T) {
+	serverName := tc.CacheName("server0")
+	toToolName := "to0"
+	toURL := "trafficops.example.net"
+	atsMajorVersion := 7
+
+	cacheURLConfigParams := map[string]string{
+		"location": "notinconfig",
+	}
+
+	dsProfilesCacheKeyConfigParams := map[int]map[string]string{
+		49: map[string]string{
+			"cachekeykey": "cachekeyval",
+		},
+		44: map[string]string{
+			"shouldnotincludeotherprofile": "shouldnotincludeotherprofileval",
+		},
+	}
+
+	serverPackageParamData := map[string]string{
+		"dscp_remap_no": "notused",
+	}
+
+	serverInfo := &ServerInfo{
+		CacheGroupID:                  42,
+		CDN:                           "mycdn",
+		CDNID:                         43,
+		DomainName:                    "mydomain",
+		HostName:                      "myhost",
+		HTTPSPort:                     12443,
+		ID:                            44,
+		IP:                            "192.168.2.4",
+		ParentCacheGroupID:            45,
+		ParentCacheGroupType:          "CGType4",
+		ProfileID:                     46,
+		ProfileName:                   "MyProfile",
+		Port:                          12080,
+		SecondaryParentCacheGroupID:   47,
+		SecondaryParentCacheGroupType: "MySecondaryParentCG",
+		Type:                          "EDGE",
+	}
+
+	remapDSData := []RemapConfigDSData{
+		RemapConfigDSData{
+			ID:                       48,
+			Type:                     "HTTP_LIVE_NATNL",
+			OriginFQDN:               util.StrPtr("myorigin"),
+			MidHeaderRewrite:         util.StrPtr("mymidrewrite"),
+			CacheURL:                 util.StrPtr(""),
+			RangeRequestHandling:     util.IntPtr(tc.RangeRequestHandlingSlice),
+			CacheKeyConfigParams:     map[string]string{"cachekeyparamname": "cachekeyparamval"},
+			RemapText:                util.StrPtr("@plugin=tslua.so @pparam=my-range-manipulator.lua"),
+			EdgeHeaderRewrite:        nil,
+			SigningAlgorithm:         util.StrPtr("foo"),
+			Name:                     "mydsname",
+			QStringIgnore:            util.IntPtr(int(tc.QueryStringIgnoreIgnoreInCacheKeyAndPassUp)),
+			RegexRemap:               util.StrPtr(""),
+			FQPacingRate:             util.IntPtr(0),
+			DSCP:                     0,
+			RoutingName:              util.StrPtr("myroutingname"),
+			MultiSiteOrigin:          util.StrPtr("mymso"),
+			Pattern:                  util.StrPtr(`mypattern`),
+			RegexType:                util.StrPtr(string(tc.DSMatchTypeHostRegex)),
+			Domain:                   util.StrPtr("mydomain"),
+			RegexSetNumber:           util.StrPtr("myregexsetnum"),
+			OriginShield:             util.StrPtr("myoriginshield"),
+			ProfileID:                util.IntPtr(49),
+			Protocol:                 util.IntPtr(int(tc.DSProtocolHTTPToHTTPS)),
+			AnonymousBlockingEnabled: util.BoolPtr(false),
+			Active:                   true,
+			RangeSliceBlockSize:      util.IntPtr(262144),
+		},
+	}
+
+	txt := MakeRemapDotConfig(serverName, toToolName, toURL, atsMajorVersion, cacheURLConfigParams, dsProfilesCacheKeyConfigParams, serverPackageParamData, serverInfo, remapDSData)
+
+	txt = strings.TrimSpace(txt)
+
+	testComment(t, txt, string(serverName), toToolName, toURL)
+
+	txtLines := strings.Split(txt, "\n")
+
+	if len(txtLines) != 2 {
+		t.Fatalf("expected 1 remaps from HTTP_TO_HTTPS DS, actual: '%v' count %v", txt, len(txtLines))
+	}
+
+	remapLine := txtLines[1]
+
+	if !strings.HasPrefix(remapLine, "map") {
+		t.Errorf("expected to start with 'map', actual '%v'", txt)
+	}
+
+	if !strings.Contains(remapLine, "slice.so") {
+		t.Errorf("expected remap on edge server with ds slice range request handling to contain background fetch plugin, actual '%v'", txt)
+	}
+
+	if !strings.Contains(remapLine, "cache_range_requests.so") {
+		t.Errorf("expected remap on edge server with ds slice range request handling to contain cache_range_requests plugin, actual '%v'", txt)
+	}
+
+	if !strings.Contains(remapLine, "pparam=--blockbytes=262144") {
+		t.Errorf("expected remap on edge server with ds slice range request handling to contain block size for the slice plugin, actual '%v'", txt)
+	}
+
+	if !strings.HasSuffix(remapLine, "@plugin=tslua.so @pparam=my-range-manipulator.lua") {
+		t.Errorf("expected raw remap without range directive at end of remap line, actual '%v'", txt)
+	}
+	if strings.Count(remapLine, "slice.so") != 1 {
+		t.Errorf("expected raw remap range directive to not be duplicated, actual '%v'", txt)
+	}
+}
+
 func TestMakeRemapDotConfigEdgeRangeRequestCache(t *testing.T) {
 	serverName := tc.CacheName("server0")
 	toToolName := "to0"