Configure uncacheable delivery services in Varnish (#7771)

lib/go-atscfg/cachedotconfig.go

@@ -20,6 +20,7 @@ package atscfg
  */
 
 import (
+	"fmt"
 	"sort"
 	"strings"
 
@@ -71,6 +72,62 @@ func makeCacheDotConfigEdge(
 		return Cfg{}, makeErr(warnings, "server missing profiles")
 	}
 
+	profileDSes, dsWarnings := GetProfileDSes(server, servers, deliveryServices, deliveryServiceServers)
+	warnings = append(warnings, dsWarnings...)
+
+	lines := map[string]struct{}{} // use a "set" for lines, to avoid duplicates, since we're looking up by profile
+	for _, ds := range profileDSes {
+		if ds.Type != tc.DSTypeHTTPNoCache {
+			continue
+		}
+		if ds.OriginFQDN == "" {
+			warnings = append(warnings, fmt.Sprintf("profileCacheDotConfig ds %s has no origin fqdn, skipping!", ds.Name))
+			continue
+		}
+		originFQDN, originPort := GetHostPortFromURI(ds.OriginFQDN)
+		if originPort != "" {
+			l := "dest_domain=" + originFQDN + " port=" + originPort + " scheme=http action=never-cache\n"
+			lines[l] = struct{}{}
+		} else {
+			l := "dest_domain=" + originFQDN + " scheme=http action=never-cache\n"
+			lines[l] = struct{}{}
+		}
+	}
+
+	linesArr := []string{}
+	for line, _ := range lines {
+		linesArr = append(linesArr, line)
+	}
+	sort.Strings(linesArr)
+	text := strings.Join(linesArr, "")
+	if text == "" {
+		text = "\n" // If no params exist, don't send "not found," but an empty file. We know the profile exists.
+	}
+	hdr := makeHdrComment(opt.HdrComment)
+	text = hdr + text
+
+	return Cfg{
+		Text:        text,
+		ContentType: ContentTypeCacheDotConfig,
+		LineComment: LineCommentCacheDotConfig,
+		Warnings:    warnings,
+	}, nil
+}
+
+// ProfileDS struct for filtered delivery services.
+type ProfileDS struct {
+	Name       string
+	Type       tc.DSType
+	OriginFQDN string
+}
+
+// GetProfileDSes filters delivery services and return delivery services with valid type and non-empty FQDN.
+func GetProfileDSes(server *Server,
+	servers []Server,
+	deliveryServices []DeliveryService,
+	deliveryServiceServers []DeliveryServiceServer,
+) ([]ProfileDS, []string) {
+	warnings := make([]string, 0)
 	profileServerIDsMap := map[int]struct{}{}
 	for _, sv := range servers {
 		if len(sv.ProfileNames) == 0 {
@@ -97,7 +154,7 @@ func makeCacheDotConfigEdge(
 		dsIDs[dss.DeliveryService] = struct{}{}
 	}
 
-	profileDSes := []profileDS{}
+	profileDSes := []ProfileDS{}
 	for _, ds := range deliveryServices {
 		if ds.ID == nil {
 			warnings = append(warnings, "deliveryservices had ds with nil id, skipping!")
@@ -110,6 +167,10 @@ func makeCacheDotConfigEdge(
 		if ds.OrgServerFQDN == nil {
 			continue // this is normal for steering and anymap dses
 		}
+		if ds.XMLID == nil || *ds.XMLID == "" {
+			warnings = append(warnings, "got ds with missing XMLID, skipping!")
+			continue
+		}
 		if *ds.Type == tc.DSTypeInvalid {
 			warnings = append(warnings, "deliveryservices had ds with invalid type, skipping!")
 			continue
@@ -121,73 +182,31 @@ func makeCacheDotConfigEdge(
 		if _, ok := dsIDs[*ds.ID]; !ok && ds.Topology == nil {
 			continue
 		}
-		origin := *ds.OrgServerFQDN
-		profileDSes = append(profileDSes, profileDS{Type: *ds.Type, OriginFQDN: &origin})
+		profileDSes = append(profileDSes, ProfileDS{Name: *ds.XMLID, Type: *ds.Type, OriginFQDN: *ds.OrgServerFQDN})
 	}
 
-	lines := map[string]struct{}{} // use a "set" for lines, to avoid duplicates, since we're looking up by profile
-	for _, ds := range profileDSes {
-		if ds.Type != tc.DSTypeHTTPNoCache {
-			continue
-		}
-		if ds.OriginFQDN == nil || *ds.OriginFQDN == "" {
-			warnings = append(warnings, "profileCacheDotConfig ds has no origin fqdn, skipping!") // TODO add ds name to data loaded, to put it in the error here?
-			continue
-		}
-		originFQDN, originPort := getHostPortFromURI(*ds.OriginFQDN)
-		if originPort != "" {
-			l := "dest_domain=" + originFQDN + " port=" + originPort + " scheme=http action=never-cache\n"
-			lines[l] = struct{}{}
-		} else {
-			l := "dest_domain=" + originFQDN + " scheme=http action=never-cache\n"
-			lines[l] = struct{}{}
-		}
-	}
-
-	linesArr := []string{}
-	for line, _ := range lines {
-		linesArr = append(linesArr, line)
-	}
-	sort.Strings(linesArr)
-	text := strings.Join(linesArr, "")
-	if text == "" {
-		text = "\n" // If no params exist, don't send "not found," but an empty file. We know the profile exists.
-	}
-	hdr := makeHdrComment(opt.HdrComment)
-	text = hdr + text
-
-	return Cfg{
-		Text:        text,
-		ContentType: ContentTypeCacheDotConfig,
-		LineComment: LineCommentCacheDotConfig,
-		Warnings:    warnings,
-	}, nil
-}
-
-type profileDS struct {
-	Type       tc.DSType
-	OriginFQDN *string
+	return profileDSes, warnings
 }
 
 // dsesToProfileDSes is a helper function to convert a []tc.DeliveryServiceNullable to []ProfileDS.
 // Note this does not check for nil values. If any DeliveryService's Type or OrgServerFQDN may be nil, the returned ProfileDS should be checked for DSTypeInvalid and nil, respectively.
-func dsesToProfileDSes(dses []tc.DeliveryServiceNullable) []profileDS {
-	pdses := []profileDS{}
+func dsesToProfileDSes(dses []tc.DeliveryServiceNullable) []ProfileDS {
+	pdses := []ProfileDS{}
 	for _, ds := range dses {
-		pds := profileDS{}
+		pds := ProfileDS{}
 		if ds.Type != nil {
 			pds.Type = *ds.Type
 		}
 		if ds.OrgServerFQDN != nil && *ds.OrgServerFQDN != "" {
-			org := *ds.OrgServerFQDN
-			pds.OriginFQDN = &org
+			pds.OriginFQDN = *ds.OrgServerFQDN
 		}
 		pdses = append(pdses, pds)
 	}
 	return pdses
 }
 
-func getHostPortFromURI(uriStr string) (string, string) {
+// GetHostPortFromURI strips HTTP(s) scheme and path and return host with port (if found).
+func GetHostPortFromURI(uriStr string) (string, string) {
 	originFQDN := uriStr
 	originFQDN = strings.TrimPrefix(originFQDN, "http://")
 	originFQDN = strings.TrimPrefix(originFQDN, "https://")

lib/varnishcfg/cache_control.go

@@ -0,0 +1,57 @@
+package varnishcfg
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/apache/trafficcontrol/lib/go-atscfg"
+	"github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+func (v VCLBuilder) configureUncacheableDSes(vclFile *vclFile, profileDSes []atscfg.ProfileDS) []string {
+	warnings := make([]string, 0)
+
+	uncacheableHostsConditions := make([]string, 0)
+	for _, ds := range profileDSes {
+		if ds.Type != tc.DSTypeHTTPNoCache {
+			continue
+		}
+
+		if ds.OriginFQDN == "" {
+			warnings = append(warnings, fmt.Sprintf("ds %s has no origin fqdn, skipping!", ds.Name))
+			continue
+		}
+
+		host, _ := atscfg.GetHostPortFromURI(ds.OriginFQDN)
+		uncacheableHostsConditions = append(uncacheableHostsConditions, fmt.Sprintf(`bereq.http.host == "%s"`, host))
+	}
+	if len(uncacheableHostsConditions) == 0 {
+		return warnings
+	}
+	berespLines := make([]string, 0)
+	berespLines = append(berespLines, fmt.Sprintf(`if (%s) {`, strings.Join(uncacheableHostsConditions, " || ")))
+	berespLines = append(berespLines, fmt.Sprint(`	set beresp.uncacheable = true;`))
+	berespLines = append(berespLines, fmt.Sprint(`}`))
+	vclFile.subroutines["vcl_backend_response"] = append(vclFile.subroutines["vcl_backend_response"], berespLines...)
+
+	return warnings
+}

lib/varnishcfg/cache_control_test.go

@@ -0,0 +1,90 @@
+package varnishcfg
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/apache/trafficcontrol/cache-config/t3cutil"
+	"github.com/apache/trafficcontrol/lib/go-atscfg"
+	"github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+func TestConfigureUncacheableDSes(t *testing.T) {
+	testCases := []struct {
+		name                string
+		dses                []atscfg.ProfileDS
+		expectedSubroutines map[string][]string
+	}{
+		{
+			name: "no uncacheable DSes",
+			dses: []atscfg.ProfileDS{
+				{Name: "ds1", Type: tc.DSTypeHTTP, OriginFQDN: "ds1.example.com"},
+				{Name: "ds2", Type: tc.DSTypeHTTP, OriginFQDN: "ds2.example.com"},
+			},
+			expectedSubroutines: make(map[string][]string),
+		},
+		{
+			name: "single uncacheable DS",
+			dses: []atscfg.ProfileDS{
+				{Name: "ds1", Type: tc.DSTypeHTTP, OriginFQDN: "ds1.example.com"},
+				{Name: "ds2", Type: tc.DSTypeHTTPNoCache, OriginFQDN: "ds2.example.com"},
+			},
+			expectedSubroutines: map[string][]string{
+				"vcl_backend_response": {
+					`if (bereq.http.host == "ds2.example.com") {`,
+					`	set beresp.uncacheable = true;`,
+					`}`,
+				},
+			},
+		},
+		{
+			name: "multiple uncacheable DS",
+			dses: []atscfg.ProfileDS{
+				{Name: "ds1", Type: tc.DSTypeHTTP, OriginFQDN: "ds1.example.com"},
+				{Name: "ds2", Type: tc.DSTypeHTTPNoCache, OriginFQDN: "ds2.example.com"},
+				{Name: "ds3", Type: tc.DSTypeHTTPNoCache, OriginFQDN: "ds3.example.com"},
+				{Name: "ds4", Type: tc.DSTypeHTTPNoCache, OriginFQDN: "ds4.example.com"},
+				{Name: "ds5", Type: tc.DSTypeDNS, OriginFQDN: "ds5.example.com"},
+			},
+			expectedSubroutines: map[string][]string{
+				"vcl_backend_response": {
+					`if (bereq.http.host == "ds2.example.com" || bereq.http.host == "ds3.example.com" || bereq.http.host == "ds4.example.com") {`,
+					`	set beresp.uncacheable = true;`,
+					`}`,
+				},
+			},
+		},
+	}
+	for _, tC := range testCases {
+		t.Run(tC.name, func(t *testing.T) {
+			vb := NewVCLBuilder(&t3cutil.ConfigData{})
+			vclFile := newVCLFile(defaultVCLVersion)
+			warnings := vb.configureUncacheableDSes(&vclFile, tC.dses)
+			if len(warnings) != 0 {
+				t.Errorf("got unexpected warnings %v", warnings)
+			}
+			if !reflect.DeepEqual(vclFile.subroutines, tC.expectedSubroutines) {
+				t.Errorf("got %v want %v", vclFile.subroutines, tC.expectedSubroutines)
+			}
+		})
+	}
+}

lib/varnishcfg/vclbuilder.go

@@ -75,8 +75,18 @@ func (vb *VCLBuilder) BuildVCLFile() (string, []string, error) {
 	if err != nil {
 		return "", nil, fmt.Errorf("(warnings: %s) %w", strings.Join(warnings, ", "), err)
 	}
+	profileDSes, dsWarnings := atscfg.GetProfileDSes(
+		vb.toData.Server,
+		vb.toData.Servers,
+		vb.toData.DeliveryServices,
+		vb.toData.DeliveryServiceServers,
+	)
+	warnings = append(warnings, dsWarnings...)
+	cacheWarnings := vb.configureUncacheableDSes(&v, profileDSes)
+	warnings = append(warnings, cacheWarnings...)
 
 	dirWarnings, err := vb.configureDirectors(&v, parents)
 	warnings = append(warnings, dirWarnings...)
+
 	return fmt.Sprint(v), warnings, err
 }