[WIP] Add Ratelimiting test using Envoy's Ratelimit Service

tests/integration/policy/ratelimiting_test.go

@@ -0,0 +1,189 @@
+//  Copyright 2020 Istio Authors
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package policy
+
+import (
+	"io/ioutil"
+	"net/http"
+
+	"math"
+	"strings"
+	"testing"
+	"time"
+
+	"istio.io/istio/pkg/test/framework"
+	"istio.io/istio/pkg/test/framework/components/bookinfo"
+	"istio.io/istio/pkg/test/framework/components/galley"
+	"istio.io/istio/pkg/test/framework/components/ingress"
+	"istio.io/istio/pkg/test/framework/components/istio"
+	"istio.io/istio/pkg/test/framework/components/namespace"
+	"istio.io/istio/pkg/test/framework/label"
+	"istio.io/istio/pkg/test/framework/resource"
+	"istio.io/istio/pkg/test/framework/resource/environment"
+	util "istio.io/istio/tests/integration/mixer"
+)
+
+var (
+	ist         istio.Instance
+	bookinfoNs  namespace.Instance
+	ratelimitNs namespace.Instance
+	g           galley.Instance
+	ing         ingress.Instance
+)
+
+func TestRateLimiting_DefaultLessThanOverride(t *testing.T) {
+	framework.
+		NewTest(t).
+		RequiresEnvironment(environment.Kube).
+		Run(func(ctx framework.TestContext) {
+			util.AllowRuleSync(t)
+
+			err := setupEnvoyFilter(ratelimitNs, ctx)
+			if err != nil {
+				return
+			}
+
+			res := util.SendTraffic(ing, t, "Sending traffic...", "", "", 300)
+			totalReqs := float64(res.DurationHistogram.Count)
+			got429s := float64(res.RetCodes[http.StatusTooManyRequests])
+			actualDuration := res.ActualDuration.Seconds() // can be a bit more than requested
+
+			// Sending 10 requests. Ratelimiting is set to 1 request per minute for productpage.
+			want200s := 1.0
+			// everything in excess of 200s should be 429s (ideally)
+			want429s := totalReqs - want200s
+			t.Logf("Expected Totals: 200s: %f (%f rps), 429s: %f (%f rps)", want200s, want200s/actualDuration,
+				want429s, want429s/actualDuration)
+
+			// check resource exhausted
+			want429s = math.Floor(want429s * 0.97)
+			if got429s < want429s {
+				t.Errorf("Bad metric value for rate-limited requests (429s): got %f, want at least %f", got429s,
+					want429s)
+			}
+		})
+}
+
+func TestMain(m *testing.M) {
+	framework.
+		NewSuite("envoy_policy_ratelimit", m).
+		Label(label.CustomSetup).
+		RequireEnvironment(environment.Kube).
+		RequireSingleCluster().
+		SetupOnEnv(environment.Kube, istio.Setup(&ist, setupConfig)).
+		Setup(testsetup).
+		Run()
+}
+
+func setupConfig(cfg *istio.Config) {
+	if cfg == nil {
+		return
+	}
+	cfg.Values["telemetry.enabled"] = "true"
+	cfg.Values["telemetry.v1.enabled"] = "false"
+	cfg.Values["telemetry.v2.enabled"] = "true"
+}
+
+func testsetup(ctx resource.Context) (err error) {
+	bookinfoNs, err = namespace.New(ctx, namespace.Config{
+		Prefix: "istio-bookinfo",
+		Inject: true,
+	})
+	if err != nil {
+		return
+	}
+	if _, err = bookinfo.Deploy(ctx, bookinfo.Config{Namespace: bookinfoNs, Cfg: bookinfo.BookInfo}); err != nil {
+		return
+	}
+	g, err = galley.New(ctx, galley.Config{})
+	if err != nil {
+		return
+	}
+
+	ing, err = ingress.New(ctx, ingress.Config{Istio: ist})
+	if err != nil {
+		return
+	}
+
+	bookinfoGatewayFile, err := bookinfo.NetworkingBookinfoGateway.LoadGatewayFileWithNamespace(bookinfoNs.Name())
+	if err != nil {
+		return
+	}
+	destinationRule, err := bookinfo.GetDestinationRuleConfigFile(ctx)
+	if err != nil {
+		return
+	}
+	destinationRuleFile, err := destinationRule.LoadWithNamespace(bookinfoNs.Name())
+	if err != nil {
+		return
+	}
+	virtualServiceFile, err := bookinfo.NetworkingVirtualServiceAllV1.LoadWithNamespace(bookinfoNs.Name())
+	if err != nil {
+		return
+	}
+	err = g.ApplyConfig(bookinfoNs,
+		bookinfoGatewayFile,
+		destinationRuleFile,
+		virtualServiceFile)
+	if err != nil {
+		return
+	}
+
+	//TODO(gargnuur): Make this a component in test pkg framework and wait for pods to
+	// come up rather than sleep.
+	ratelimitNs, err = namespace.New(ctx, namespace.Config{
+		Prefix: "istio-ratelimit",
+	})
+	if err != nil {
+		return
+	}
+
+	yamlContent, err := ioutil.ReadFile("testdata/ratelimitservice.yaml")
+	if err != nil {
+		return
+	}
+
+	err = g.ApplyConfig(ratelimitNs,
+		string(yamlContent),
+	)
+	if err != nil {
+		return
+	}
+
+	time.Sleep(time.Second * 30)
+
+	return nil
+}
+
+func setupEnvoyFilter(ratelimitNs namespace.Instance, ctx resource.Context) error {
+	content, err := ioutil.ReadFile("testdata/enable_envoy_ratelimit.yaml")
+	if err != nil {
+		return err
+	}
+	con := string(content)
+
+	con = strings.Replace(con, "ratelimit.default.svc.cluster.local",
+		"ratelimit."+ratelimitNs.Name()+".svc.cluster.local", -1)
+
+	ns, err := namespace.Claim(ctx, ist.Settings().SystemNamespace, true)
+	if err != nil {
+		return err
+	}
+	err = g.ApplyConfig(ns, con)
+	if err != nil {
+		return err
+	}
+	return nil
+}

tests/integration/policy/testdata/enable_envoy_ratelimit.yaml

@@ -0,0 +1,82 @@
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+  name: filter-ratelimit
+  namespace: istio-system
+spec:
+  workloadSelector:
+    # select by label in the same namespace
+    labels:
+      istio: ingressgateway
+  configPatches:
+    # The Envoy config you want to modify
+    - applyTo: HTTP_FILTER
+      match:
+        context: GATEWAY
+        listener:
+          filterChain:
+            filter:
+              name: "envoy.http_connection_manager"
+              subFilter:
+                name: "envoy.router"
+      patch:
+        operation: INSERT_BEFORE
+        value:
+          name: envoy.rate_limit
+          config:
+            # domain can be anything! Match it to the ratelimter service config
+            domain: productpage-ratelimit
+            failure_mode_deny: true
+            rate_limit_service:
+              grpc_service:
+                envoy_grpc:
+                  cluster_name: rate_limit_cluster
+                timeout: 10s
+    - applyTo: CLUSTER
+      match:
+        cluster:
+          service: ratelimit.default.svc.cluster.local
+      patch:
+        operation: ADD
+        value:
+          name: rate_limit_cluster
+          type: STRICT_DNS
+          connect_timeout: 10s
+          lb_policy: ROUND_ROBIN
+          http2_protocol_options: {}
+          hosts:
+          - socket_address:
+              address: ratelimit.default.svc.cluster.local
+              port_value: 8081
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+  name: filter-ratelimit-svc
+  namespace: istio-system
+spec:
+  workloadSelector:
+    labels:
+      istio: ingressgateway
+  configPatches:
+    - applyTo: VIRTUAL_HOST
+      match:
+        context: GATEWAY
+        routeConfiguration:
+          vhost:
+            name: "*:80"
+            route:
+              action: ANY
+      patch:
+        operation: MERGE
+        value:
+          rate_limits:
+            - actions: # any actions in here
+                  # Multiple actions nest the descriptors
+                  # - generic_key:
+                  # descriptor_value: "test"
+              - request_headers:
+                  header_name: ":path"
+                  descriptor_key: "PATH"
+                  # - remote_address: {}
+                  # - destination_cluster: {}

tests/integration/policy/testdata/ratelimitservice.yaml

@@ -0,0 +1,112 @@
+# Copyright 2020 Istio Authors
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+  labels:
+    app: redis
+spec:
+  ports:
+  - name: redis
+    port: 6379
+  selector:
+    app: redis
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+      - image: redis:alpine
+        imagePullPolicy: Always
+        name: redis
+        ports:
+        - name: redis
+          containerPort: 6379
+      restartPolicy: Always
+      serviceAccountName: ""
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ratelimit
+  labels:
+    app: ratelimit
+spec:
+  ports:
+  - name: "8080"
+    port: 8080
+    targetPort: 8080
+    protocol: TCP
+  - name: "8081"
+    port: 8081
+    targetPort: 8081
+    protocol: TCP
+  - name: "6070"
+    port: 6070
+    targetPort: 6070
+    protocol: TCP
+  selector:
+    app: ratelimit
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ratelimit
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ratelimit
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: ratelimit
+    spec:
+      containers:
+      # TODO(gargnupur): Upload this to istio-testing
+      - image: gcr.io/gargnupur-istio-test/ratelimit:v1.4.0
+        imagePullPolicy: Always
+        name: ratelimit
+        env:
+        - name: LOG_LEVEL
+          value: debug
+        - name: REDIS_SOCKET_TYPE
+          value: tcp
+        - name: REDIS_URL
+          value: redis:6379
+        - name: USE_STATSD
+          value: "false"
+        - name: RUNTIME_ROOT
+          value: /data
+        - name: RUNTIME_SUBDIRECTORY
+          value: ratelimit
+        ports:
+        - containerPort: 8080
+        - containerPort: 8081
+        - containerPort: 6070
+---