Merge branch 'release-1.1' of https://github.com/istio/istio into hea…

…lth-autoinject

.circleci/config.yml

@@ -236,7 +236,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_yaml
+            make docker.all generate_e2e_test_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:
@@ -272,7 +272,7 @@ jobs:
               # Should only happen when re-running a job, and the workspace is gone
               time make build test-bins
             fi
-            make docker.all generate_yaml
+            make docker.all generate_e2e_test_yaml
       - run: bin/testEnvRootMinikube.sh wait
       - run: docker images
       - run:
@@ -485,7 +485,7 @@ jobs:
               export PATH=$GOPATH/bin:$PATH
               make localTestEnv
               set -o pipefail
-              make test.integration T=-v | tee -a /go/out/tests/build-log.txt
+              make test.integration.local T=-v | tee -a /go/out/tests/build-log.txt
         - <<: *recordZeroExitCodeIfTestPassed
         - <<: *recordNonzeroExitCodeIfTestFailed
         - <<: *markJobFinishesOnGCS

.codecov.yml

@@ -3,6 +3,8 @@ coverage:
   round: up
   range: 60..99
   ignore:
+    - "mixer/test"
+    - "mixer/template"
     - "**/*.pb.go" # Auto-generated proto files
     - "tests/" # Test infrastructure coverage does not affect core coverage
     - "**/test/*.go"

codecov.skip

@@ -4,7 +4,6 @@ istio.io/istio/mixer/pkg/mockapi
 istio.io/istio/mixer/pkg/perf
 istio.io/istio/mixer/pkg/runtime/testing
 istio.io/istio/mixer/template/sample
-istio.io/istio/mixer/test
 istio.io/istio/mixer/tools/codegen
 istio.io/istio/pilot/test
 istio.io/istio/pkg/mcp/testing

galley/pkg/crd/validation/endpoint.go

@@ -39,7 +39,10 @@ func endpointReady(store cache.KeyGetter, queue workqueue.RateLimitingInterface,
 	if err != nil || !exists {
 		return endpointCheckNotReady
 	}
-	endpoints := item.(*v1.Endpoints)
+	endpoints, ok := item.(*v1.Endpoints)
+	if !ok {
+		return endpointCheckNotReady
+	}
 	if len(endpoints.Subsets) == 0 {
 		scope.Warnf("%s/%v endpoint not ready: no subsets", namespace, name)
 		return endpointCheckNotReady
@@ -65,9 +68,6 @@ func (wh *Webhook) waitForEndpointReady(stopCh <-chan struct{}) (shutdown bool)
 		}
 	}()
 
-	controllerStopCh := make(chan struct{})
-	defer close(controllerStopCh)
-
 	queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
 	defer queue.ShutDown()
 
@@ -97,21 +97,30 @@ func (wh *Webhook) waitForEndpointReady(stopCh <-chan struct{}) (shutdown bool)
 			},
 		},
 	)
-	go controller.Run(stopCh)
+
+	controllerStopCh := make(chan struct{})
+	defer close(controllerStopCh)
+	go controller.Run(controllerStopCh)
 
 	if !cache.WaitForCacheSync(stopCh, controller.HasSynced) {
+		scope.Errorf("wait for cache sync failed")
 		return true
 	}
 
 	for {
-		ready := endpointReady(store, queue, wh.deploymentAndServiceNamespace, wh.serviceName)
-		switch ready {
-		case endpointCheckShutdown:
+		select {
+		case <-stopCh:
 			return true
-		case endpointCheckReady:
-			return false
-		case endpointCheckNotReady:
-			// continue waiting for endpoint to be ready
+		default:
+			ready := endpointReady(store, queue, wh.deploymentAndServiceNamespace, wh.serviceName)
+			switch ready {
+			case endpointCheckShutdown:
+				return true
+			case endpointCheckReady:
+				return false
+			case endpointCheckNotReady:
+				// continue waiting for endpoint to be ready
+			}
 		}
 	}
 }

galley/pkg/crd/validation/validation.go

@@ -70,7 +70,7 @@ func createMixerValidator() store.BackendValidator {
 func webhookHTTPSHandlerReady(client httpClient, vc *WebhookParameters) error {
 	readinessURL := &url.URL{
 		Scheme: "https",
-		Host:   fmt.Sprintf("localhost:%v", vc.Port),
+		Host:   fmt.Sprintf("127.0.0.1:%v", vc.Port),
 		Path:   httpsHandlerReadyPath,
 	}
 

install/kubernetes/helm/istio-remote/templates/sidecar-injector-configmap.yaml

@@ -13,6 +13,7 @@ metadata:
 data:
   config: |-
     policy: {{ .Values.global.proxy.autoInject }}
+    rewriteAppHTTPProbe: {{ .Values.sidecarInjectorWebhook.rewriteAppHTTPProbe }}
     template: |-
 {{- if or (not .Values.istio_cni.enabled) .Values.global.proxy.enableCoreDump }}
       initContainers:
@@ -170,7 +171,7 @@ data:
             - NET_ADMIN
           runAsGroup: 1337
           {{ "[[ else -]]" }}
-          {{ if and .Values.global.sds.enabled .Values.global.sds.enableTokenMount }}
+          {{ if and .Values.global.sds.enabled .Values.global.sds.useTrustworthyJwt }}
           runAsGroup: 1337
           {{- end }}
           runAsUser: 1337
@@ -194,7 +195,7 @@ data:
         {{- if .Values.global.sds.enabled }}
         - mountPath: /var/run/sds
           name: sds-uds-path
-        {{- if .Values.global.sds.enableTokenMount }}
+        {{- if .Values.global.sds.useTrustworthyJwt }}
         - mountPath: /var/run/secrets/tokens
           name: istio-token
         {{- end }}
@@ -204,7 +205,7 @@ data:
       - name: sds-uds-path
         hostPath:
           path: /var/run/sds
-      {{- if .Values.global.sds.enableTokenMount }}
+      {{- if .Values.global.sds.useTrustworthyJwt }}
       - name: istio-token
         projected:
           sources:

install/kubernetes/helm/istio-remote/values.yaml

@@ -258,7 +258,8 @@ global:
   sds:
     enabled: false
     udsPath: ""
-    enableTokenMount: false
+    useTrustworthyJwt: false
+    useNormalJwt: false
 
   # Sets an identifier for the remote network to be used for Split Horizon EDS. The network will be sent
   # to the Pilot when connected by the sidecar and will affect the results returned in EDS requests.

install/kubernetes/helm/istio/templates/configmap.yaml

@@ -60,7 +60,15 @@ data:
     # If set to true(prerequisite: https://kubernetes.io/docs/concepts/storage/volumes/#projected), Istio will inject volumes mount 
     # for k8s service account JWT, so that K8s API server mounts k8s service account JWT to envoy container, which 
     # will be used to generate key/cert eventually. This isn't supported for non-k8s case.
-    enableSdsTokenMount: {{ .Values.global.sds.enableTokenMount }}
+    enableSdsTokenMount: {{ .Values.global.sds.useTrustworthyJwt }}
+
+    # This flag is used by secret discovery service(SDS). 
+    # If set to true, envoy will fetch normal k8s service account JWT from '/var/run/secrets/kubernetes.io/serviceaccount/token' 
+    # (https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod) 
+    # and pass to sds server, which will be used to request key/cert eventually. 
+    # this flag is ignored if enableSdsTokenMount is set.
+    # This isn't supported for non-k8s case.
+    sdsUseK8sSaJwt: {{ .Values.global.sds.useNormalJwt }}
 
     # The trust domain corresponds to the trust root of a system.
     # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain

install/kubernetes/helm/istio/templates/sidecar-injector-configmap.yaml

@@ -14,6 +14,7 @@ data:
   config: |-
     policy: {{ .Values.global.proxy.autoInject }}
     template: |-
+      rewriteAppHTTPProbe: {{ .Values.sidecarInjectorWebhook.rewriteAppHTTPProbe }}
 {{- if or (not .Values.istio_cni.enabled) .Values.global.proxy.enableCoreDump }}
       initContainers:
 {{- if not .Values.istio_cni.enabled }}
@@ -202,7 +203,7 @@ data:
             - NET_ADMIN
           runAsGroup: 1337
           {{ "[[ else -]]" }}
-          {{ if and .Values.global.sds.enabled .Values.global.sds.enableTokenMount }}
+          {{ if and .Values.global.sds.enabled .Values.global.sds.useTrustworthyJwt }}
           runAsGroup: 1337
           {{- end }}
           runAsUser: 1337
@@ -226,7 +227,7 @@ data:
         {{- if .Values.global.sds.enabled }}
         - mountPath: /var/run/sds
           name: sds-uds-path
-        {{- if .Values.global.sds.enableTokenMount }}
+        {{- if .Values.global.sds.useTrustworthyJwt }}
         - mountPath: /var/run/secrets/tokens
           name: istio-token
         {{- end }}
@@ -241,7 +242,7 @@ data:
       - name: sds-uds-path
         hostPath:
           path: /var/run/sds
-      {{- if .Values.global.sds.enableTokenMount }}
+      {{- if .Values.global.sds.useTrustworthyJwt }}
       - name: istio-token
         projected:
           sources:

install/kubernetes/helm/istio/values.yaml

@@ -11,6 +11,7 @@ gateways:
 #
 sidecarInjectorWebhook:
   enabled: true
+  rewriteAppHTTPProbe: false
 
 #
 # galley configuration, refer to charts/galley/values.yaml
@@ -348,7 +349,8 @@ global:
     # distributed through the SecretDiscoveryService instead of using K8S secrets to mount the certificates.
     enabled: false
     udsPath: ""
-    enableTokenMount: false
+    useTrustworthyJwt: false
+    useNormalJwt: false
 
   # Configure the mesh networks to be used by the Split Horizon EDS.
   # 

install/kubernetes/helm/subcharts/gateways/templates/deployment.yaml

@@ -162,7 +162,7 @@ spec:
           {{- if $.Values.global.sds.enabled }}
           - name: sdsudspath
             mountPath: /var/run/sds
-          {{- if $.Values.global.sds.enableTokenMount }}
+          {{- if $.Values.global.sds.useTrustworthyJwt }}
           - name: istio-token
             mountPath: /var/run/secrets/tokens
           {{- end }}
@@ -183,7 +183,7 @@ spec:
       - name: sdsudspath
         hostPath:
           path: /var/run/sds
-      {{- if $.Values.global.sds.enableTokenMount }}
+      {{- if $.Values.global.sds.useTrustworthyJwt }}
       - name: istio-token
         projected:
           sources:

istioctl/cmd/istioctl/kubeinject.go

@@ -128,6 +128,7 @@ var (
 	verbosity                    int
 	versionStr                   string // override build version
 	enableCoreDump               bool
+	rewriteAppHTTPProbe          bool
 	imagePullPolicy              string
 	statusPort                   int
 	readinessInitialDelaySeconds uint32
@@ -274,6 +275,7 @@ istioctl kube-inject -f deployment.yaml -o deployment-injected.yaml --injectConf
 				if sidecarTemplate, err = inject.GenerateTemplateFromParams(&inject.Params{
 					InitImage:                    inject.InitImageName(hub, tag, debugMode),
 					ProxyImage:                   inject.ProxyImageName(hub, tag, debugMode),
+					RewriteAppHTTPProbe:          rewriteAppHTTPProbe,
 					Verbosity:                    verbosity,
 					SidecarProxyUID:              sidecarProxyUID,
 					Version:                      versionStr,
@@ -372,6 +374,8 @@ func init() {
 	injectCmd.PersistentFlags().BoolVar(&enableCoreDump, "coreDump",
 		true, "Enable/Disable core dumps in injected Envoy sidecar (--coreDump=true affects "+
 			"all pods in a node and should only be used the cluster admin)")
+	injectCmd.PersistentFlags().BoolVar(&rewriteAppHTTPProbe, "rewriteAppProbe", false, "Whether injector "+
+		"rewrites the liveness health check to let kubelet health check the app when mtls is on.")
 	injectCmd.PersistentFlags().StringVar(&imagePullPolicy, "imagePullPolicy", inject.DefaultImagePullPolicy,
 		"Sets the container image pull policy. Valid options are Always,IfNotPresent,Never."+
 			"The default policy is IfNotPresent.")

mixer/pkg/il/testing/tests.go

@@ -306,6 +306,29 @@ end`,
 		R:    true,
 		conf: exprEvalAttrs,
 	},
+	{
+		E:    `request.headers[toLower(source.uid)] == "curlish"`,
+		Type: descriptor.BOOL,
+		I: map[string]interface{}{
+			"request.headers": map[string]string{
+				"user-agent": "curlish",
+			},
+			"source.uid": "uSeR-agEnT",
+		},
+		R:    true,
+		conf: istio06AttributeSet,
+	},
+	{
+		E:    `request.headers[toLower("USER-AGENT")] == "curlish"`,
+		Type: descriptor.BOOL,
+		I: map[string]interface{}{
+			"request.headers": map[string]string{
+				"user-agent": "curlish",
+			},
+		},
+		R:    true,
+		conf: istio06AttributeSet,
+	},
 	{
 		E:    `match(request.headers["user-agent"], "curl*")`,
 		Type: descriptor.BOOL,

mixer/pkg/lang/ast/expr.go

@@ -369,7 +369,7 @@ func process(ex ast.Expr, tgt *Expression) (err error) {
 
 	case *ast.IndexExpr:
 		// accessing a map
-		// request.header["abc"]
+		// request.headers["abc"]
 		tgt.Fn = &Function{Name: tMap[token.LBRACK]}
 		if err = processFunc(tgt.Fn, []ast.Expr{v.X, v.Index}); err != nil {
 			return

mixer/pkg/lang/externs.go

@@ -50,6 +50,7 @@ var Externs = map[string]interpreter.Extern{
 	"endsWith":          interpreter.ExternFromFn("endsWith", externEndsWith),
 	"emptyStringMap":    interpreter.ExternFromFn("emptyStringMap", externEmptyStringMap),
 	"conditionalString": interpreter.ExternFromFn("conditionalString", externConditionalString),
+	"toLower":           interpreter.ExternFromFn("toLower", externToLower),
 }
 
 // ExternFunctionMetadata is the type-metadata about externs. It gets used during compilations.
@@ -115,6 +116,11 @@ var ExternFunctionMetadata = []ast.FunctionMetadata{
 		ReturnType:    config.STRING,
 		ArgumentTypes: []config.ValueType{config.BOOL, config.STRING, config.STRING},
 	},
+	{
+		Name:          "toLower",
+		ReturnType:    config.STRING,
+		ArgumentTypes: []config.ValueType{config.STRING},
+	},
 }
 
 func externIP(in string) ([]byte, error) {
@@ -335,3 +341,7 @@ func externConditionalString(condition bool, trueStr, falseStr string) string {
 	}
 	return falseStr
 }
+
+func externToLower(str string) string {
+	return strings.ToLower(str)
+}

mixer/pkg/status/status.go

@@ -67,6 +67,56 @@ func WithDeadlineExceeded(message string) rpc.Status {
 	return rpc.Status{Code: int32(rpc.DEADLINE_EXCEEDED), Message: message}
 }
 
+// WithUnknown returns an initialized status with the rpc.UNKNOWN code and the given message.
+func WithUnknown(message string) rpc.Status {
+	return rpc.Status{Code: int32(rpc.UNKNOWN), Message: message}
+}
+
+// WithNotFound returns an initialized status with the rpc.NOT_FOUND code and the given message.
+func WithNotFound(message string) rpc.Status {
+	return rpc.Status{Code: int32(rpc.NOT_FOUND), Message: message}
+}
+
+// WithAlreadyExists returns an initialized status with the rpc.ALREADY_EXISTS code and the given message.
+func WithAlreadyExists(message string) rpc.Status {
+	return rpc.Status{Code: int32(rpc.ALREADY_EXISTS), Message: message}
+}
+
+// WithFailedPrecondition returns an initialized status with the rpc.FAILED_PRECONDITION code and the given message.
+func WithFailedPrecondition(message string) rpc.Status {
+	return rpc.Status{Code: int32(rpc.FAILED_PRECONDITION), Message: message}
+}
+
+// WithAborted returns an initialized status with the rpc.ABORTED code and the given message.
+func WithAborted(message string) rpc.Status {
+	return rpc.Status{Code: int32(rpc.ABORTED), Message: message}
+}
+
+// WithOutOfRange returns an initialized status with the rpc.OUT_OF_RANGE code and the given message.
+func WithOutOfRange(message string) rpc.Status {
+	return rpc.Status{Code: int32(rpc.OUT_OF_RANGE), Message: message}
+}
+
+// WithUnimplemented returns an initialized status with the rpc.UNIMPLEMENTED code and the given message.
+func WithUnimplemented(message string) rpc.Status {
+	return rpc.Status{Code: int32(rpc.UNIMPLEMENTED), Message: message}
+}
+
+// WithUnavailable returns an initialized status with the rpc.UNAVAILABLE code and the given message.
+func WithUnavailable(message string) rpc.Status {
+	return rpc.Status{Code: int32(rpc.UNAVAILABLE), Message: message}
+}
+
+// WithDataLoss returns an initialized status with the rpc.DATA_LOSS code and the given message.
+func WithDataLoss(message string) rpc.Status {
+	return rpc.Status{Code: int32(rpc.DATA_LOSS), Message: message}
+}
+
+// WithUnauthenticated returns an initialized status with the rpc.UNAUTHENTICATED code and the given message.
+func WithUnauthenticated(message string) rpc.Status {
+	return rpc.Status{Code: int32(rpc.UNAUTHENTICATED), Message: message}
+}
+
 // IsOK returns true is the given status has the code rpc.OK
 func IsOK(status rpc.Status) bool {
 	return status.Code == int32(rpc.OK)