An experimental support for authn/authz w/ RBAC utilizing kube-rbac-p…

…roxy

Pre-requisites:

- Build the helm binary from this commit and place it in `$GOPATH/src/k8s.io/helm/bin/helm`
- Build the tiller docker image from this commit and push to mumoshu/tiller:canary
- Build a kube-rbac-proxy image from the latest master of https://github.com/brancz/kube-rbac-proxy and tag it mumoshu/kube-rbac-proxy:v0.2.0

Usage:

```
# 1. Update the tiller deployment
$ k edit deploy tiller-deploy

# 1-1. Modify the tiller container to listen on an another port while enabling the experimental auth feature

  - args:
    - --experimental-rbac-proxy
    - --listen
    - :44137
    command:
    - /tiller
    image: mumoshu/tiller:canary

# 1-2. Add kube-rbac-proxy is a sidecar container

  - args:
    - --alsologtostderr
    - --v=10
    - --insecure-listen-address=:44134
    - --upstream=http://127.0.0.1:44137/
    - --upstream-force-h2c
    - --auth-header-fields-enabled
    - --auth-header-user-field-name=x-forwarded-user
    - --auth-header-groups-field-name=x-forwarded-groups
    image: mumoshu/kube-rbac-proxy:v0.2.0
    imagePullPolicy: IfNotPresent
    name: kube-rbac-proxy
    ports:
    - containerPort: 44134
      name: grpc
      protocol: TCP

# 2. Create a serviceaccount for testing purpose

$ k create serviceaccount permissive-sa
$ k create clusterrolebinding permissive-binding   --clusterrole=cluster-admin --serviceaccount=permissive-sa
$ secret=<permissive-sa's secret name>
$ token=$(k get secret $secret -o json | jq -r '.data.token' | base64 -D)
$ k config set-credentials permissive-sa --token $token

# 3. Create a invalid serviceaccount for testing purpose

$ kubectl config set-credentials invalid-sa --token somerandomstring

# 4. Test it!

# 4-1. A k8s user who is authenticated and authorized to to access tiller can list helm releases

$ kubensx use #=> user: permissive-sa
$ $GOPATH/src/k8s.io/helm/bin/helm list --debug --experimental-rbac-auth-proxy

# 4-2. A k8s user who is not authenticated can not access tiller
#
# Note that the error message is unintuitive at the moment!

$ kubensx use #=> user: invalid-sa
$ $GOPATH/src/k8s.io/helm/bin/helm list --debug --experimental-rbac-auth-proxy

[debug] Created tunnel using local port: '56945'

[debug] SERVER: "127.0.0.1:56945"

Error: transport: received the unexpected content-type "text/plain; charset=utf-8"
```

Exercise:

- Create a kube-rbax-procy resource-attributes file to restrict the serviceaccount to only access specific tiller APIs. Note that a gRPC call results in an HTTP/2 request to the path /<package.Type>/<method> so you can use nonResourceURLs to write policies for gRPC calls.

Future work:

- Tiller impersonates as the authenticated user

cmd/helm/helm.go

@@ -49,6 +49,12 @@ var (
 	tlsCertDefault   = "$HELM_HOME/cert.pem"
 	tlsKeyDefault    = "$HELM_HOME/key.pem"
 
+	// assumes two thins:
+	// 1. kube-rbac-proxy is installed as a sidecar of tiller and
+	//    we can reach to tiller only via kube-rbac-proxy
+	// 2. helm passes the bearer token specified in your kubeconfig to kube-rbac-proxy for delegating authn/authz
+	rbacProxyEnable bool
+
 	tillerTunnel *kube.Tunnel
 	settings     helm_env.EnvSettings
 )
@@ -289,6 +295,21 @@ func newClient() helm.Interface {
 		}
 		options = append(options, helm.WithTLS(tlscfg))
 	}
+
+	if rbacProxyEnable {
+		config, err := configForContext(settings.KubeContext)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(2)
+		}
+		if config.TLSClientConfig.KeyFile != "" {
+			fmt.Fprintln(os.Stderr, "You have specified a key file in your kubeconfig but TLS between helm and kube-rbac-proxy is not implemented yet")
+			os.Exit(1)
+		}
+		if config.BearerToken != "" {
+			options = append(options, helm.WithBearerToken(config.BearerToken))
+		}
+	}
 	return helm.NewClient(options...)
 }
 
@@ -302,5 +323,6 @@ func addFlagsTLS(cmd *cobra.Command) *cobra.Command {
 	cmd.Flags().StringVar(&tlsKeyFile, "tls-key", tlsKeyDefault, "path to TLS key file")
 	cmd.Flags().BoolVar(&tlsVerify, "tls-verify", false, "enable TLS for request and verify remote")
 	cmd.Flags().BoolVar(&tlsEnable, "tls", false, "enable TLS for request")
+	cmd.Flags().BoolVar(&rbacProxyEnable, "experimental-rbac-proxy", false, "enable authn/authz to Tiller with K8S RBAC using kube-rbac-proxy")
 	return cmd
 }

cmd/tiller/tiller.go

@@ -73,6 +73,7 @@ var (
 	enableTracing        = flag.Bool("trace", false, "enable rpc tracing")
 	store                = flag.String("storage", storageConfigMap, "storage driver to use. One of 'configmap', 'memory', or 'secret'")
 	remoteReleaseModules = flag.Bool("experimental-release", false, "enable experimental release modules")
+	rbacProxy            = flag.Bool("experimental-rbac-proxy", false, "enable experimental RBAC auth proxy to authn/authz the helm client")
 	tlsEnable            = flag.Bool("tls", tlsEnableEnvVarDefault(), "enable TLS")
 	tlsVerify            = flag.Bool("tls-verify", tlsVerifyEnvVarDefault(), "enable TLS and verify remote certificate")
 	keyFile              = flag.String("tls-key", tlsDefaultsFromEnv("tls-key"), "path to TLS private key file")
@@ -163,7 +164,11 @@ func start() {
 		}))
 	}
 
-	rootServer = tiller.NewServer(opts...)
+	f := &tiller.ServerOptsFactory{
+		AuthProxyEnabled: *rbacProxy,
+	}
+
+	rootServer = tiller.NewServer(f, opts...)
 
 	lstn, err := net.Listen("tcp", *grpcAddr)
 	if err != nil {

pkg/helm/client.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"golang.org/x/net/context"
+	"golang.org/x/oauth2"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/keepalive"
@@ -296,6 +297,13 @@ func (h *Client) connect(ctx context.Context) (conn *grpc.ClientConn, err error)
 			Time: time.Duration(30) * time.Second,
 		}),
 	}
+	if h.opts.useBearerToken {
+		bearerTokenAccess := NewTokenAccess(&oauth2.Token{
+			TokenType:   "Bearer",
+			AccessToken: h.opts.bearerToken,
+		})
+		opts = append(opts, grpc.WithPerRPCCredentials(bearerTokenAccess))
+	}
 	switch {
 	case h.opts.useTLS:
 		opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(h.opts.tlsConfig)))
@@ -308,6 +316,26 @@ func (h *Client) connect(ctx context.Context) (conn *grpc.ClientConn, err error)
 	return conn, nil
 }
 
+// oauthAccess supplies PerRPCCredentials from a given token.
+type tokenAccess struct {
+	token oauth2.Token
+}
+
+// NewOauthAccess constructs the PerRPCCredentials using a given token.
+func NewTokenAccess(token *oauth2.Token) credentials.PerRPCCredentials {
+	return tokenAccess{token: *token}
+}
+
+func (oa tokenAccess) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
+	return map[string]string{
+		"authorization": oa.token.Type() + " " + oa.token.AccessToken,
+	}, nil
+}
+
+func (oa tokenAccess) RequireTransportSecurity() bool {
+	return false
+}
+
 // Executes tiller.ListReleases RPC.
 func (h *Client) list(ctx context.Context, req *rls.ListReleasesRequest) (*rls.ListReleasesResponse, error) {
 	c, err := h.connect(ctx)

pkg/helm/option.go

@@ -40,6 +40,10 @@ type options struct {
 	host string
 	// if set dry-run helm client calls
 	dryRun bool
+	// bearer token used to authenticate against kube-rbac-proxy in front of tiller
+	bearerToken string
+	// if set enable bearer token auth on helm client calls
+	useBearerToken bool
 	// if set enable TLS on helm client calls
 	useTLS bool
 	// if set, re-use an existing name
@@ -95,6 +99,14 @@ func WithTLS(cfg *tls.Config) Option {
 	}
 }
 
+// WithBearerToken specifics the bearer token used for authenticating helm client against kube-rbac-proxy in front of tiller
+func WithBearerToken(bearerToken string) Option {
+	return func(opts *options) {
+		opts.useBearerToken = true
+		opts.bearerToken = bearerToken
+	}
+}
+
 // BeforeCall returns an option that allows intercepting a helm client rpc
 // before being sent OTA to tiller. The intercepting function should return
 // an error to indicate that the call should not proceed or nil otherwise.

pkg/tiller/server.go

@@ -33,21 +33,26 @@ import (
 // grpc library default is 4MB
 var maxMsgSize = 1024 * 1024 * 20
 
+// ServerOptsFactory creates a set of `grpc.ServerOption` to add validation, authn and authz to Tiller
+type ServerOptsFactory struct {
+	AuthProxyEnabled bool
+}
+
 // DefaultServerOpts returns the set of default grpc ServerOption's that Tiller requires.
-func DefaultServerOpts() []grpc.ServerOption {
+func (f ServerOptsFactory) DefaultServerOpts() []grpc.ServerOption {
 	return []grpc.ServerOption{
 		grpc.MaxMsgSize(maxMsgSize),
-		grpc.UnaryInterceptor(newUnaryInterceptor()),
-		grpc.StreamInterceptor(newStreamInterceptor()),
+		grpc.UnaryInterceptor(f.newUnaryInterceptor()),
+		grpc.StreamInterceptor(f.newStreamInterceptor()),
 	}
 }
 
 // NewServer creates a new grpc server.
-func NewServer(opts ...grpc.ServerOption) *grpc.Server {
-	return grpc.NewServer(append(DefaultServerOpts(), opts...)...)
+func NewServer(f *ServerOptsFactory, opts ...grpc.ServerOption) *grpc.Server {
+	return grpc.NewServer(append(f.DefaultServerOpts(), opts...)...)
 }
 
-func newUnaryInterceptor() grpc.UnaryServerInterceptor {
+func (f *ServerOptsFactory) newUnaryInterceptor() grpc.UnaryServerInterceptor {
 	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
 		if err := checkClientVersion(ctx); err != nil {
 			// whitelist GetVersion() from the version check
@@ -56,16 +61,23 @@ func newUnaryInterceptor() grpc.UnaryServerInterceptor {
 				return nil, err
 			}
 		}
+		if err := f.optionallyCheckAuthenticatedUser(ctx); err != nil {
+			return nil, err
+		}
 		return goprom.UnaryServerInterceptor(ctx, req, info, handler)
 	}
 }
 
-func newStreamInterceptor() grpc.StreamServerInterceptor {
+func (f *ServerOptsFactory) newStreamInterceptor() grpc.StreamServerInterceptor {
 	return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
-		if err := checkClientVersion(ss.Context()); err != nil {
+		ctx := ss.Context()
+		if err := checkClientVersion(ctx); err != nil {
 			log.Println(err)
 			return err
 		}
+		if err := f.optionallyCheckAuthenticatedUser(ctx); err != nil {
+			return err
+		}
 		return goprom.StreamServerInterceptor(srv, ss, info, handler)
 	}
 }
@@ -93,3 +105,34 @@ func checkClientVersion(ctx context.Context) error {
 	}
 	return nil
 }
+
+func authenticatedUserFromContext(ctx context.Context) (string, []string) {
+	user := ""
+	groups := []string{}
+	if md, ok := metadata.FromIncomingContext(ctx); ok {
+		log.Printf("Request Metadata: %v", md)
+		if v, ok := md["x-forwarded-user"]; ok && len(v) > 0 {
+			user = v[0]
+		}
+		if v, ok := md["x-forwarded-groups"]; ok && len(v) > 0 {
+			groups = strings.Split(v[0], "|")
+		}
+	}
+	return user, groups
+}
+
+func checkAuthenticatedUser(ctx context.Context) error {
+	u, g := authenticatedUserFromContext(ctx)
+	if u == "" {
+		return fmt.Errorf("unauthorized access to tiller")
+	}
+	log.Printf("Authenticated as: user=%s, groups=%s", u, strings.Join(g, ","))
+	return nil
+}
+
+func (f *ServerOptsFactory) optionallyCheckAuthenticatedUser(ctx context.Context) error {
+	if f.AuthProxyEnabled {
+		return checkAuthenticatedUser(ctx)
+	}
+	return nil
+}