From 76b14b1d01af8fae30f3e132e1acb4027826cf65 Mon Sep 17 00:00:00 2001 From: Andrew Burke <31974658+atburke@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:58:16 -0700 Subject: [PATCH] Allow Azure/IAM join over reverse tunnel (#31000) This change adds support for gRPC-based join methods (Azure and IAM) over the reverse tunnel port. --- integration/integration_test.go | 52 ++++++++++++++++++- lib/service/service.go | 90 +++++++++++++++++++++++++++++---- 2 files changed, 131 insertions(+), 11 deletions(-) diff --git a/integration/integration_test.go b/integration/integration_test.go index 6cba691c77a6d..e610e6cf6a90e 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -60,9 +60,12 @@ import ( "github.com/gravitational/teleport" "github.com/gravitational/teleport/api/breaker" + apiclient "github.com/gravitational/teleport/api/client" "github.com/gravitational/teleport/api/client/proto" "github.com/gravitational/teleport/api/constants" "github.com/gravitational/teleport/api/defaults" + apidefaults "github.com/gravitational/teleport/api/defaults" + "github.com/gravitational/teleport/api/metadata" tracessh "github.com/gravitational/teleport/api/observability/tracing/ssh" "github.com/gravitational/teleport/api/profile" "github.com/gravitational/teleport/api/types" @@ -7508,7 +7511,7 @@ func testJoinOverReverseTunnelOnly(t *testing.T, suite *integrationTestSuite) { for _, proxyProtocolEnabled := range []bool{false, true} { t.Run(fmt.Sprintf("proxy protocol: %v", proxyProtocolEnabled), func(t *testing.T) { lib.SetInsecureDevMode(true) - defer lib.SetInsecureDevMode(false) + t.Cleanup(func() { lib.SetInsecureDevMode(false) }) // Create a Teleport instance with Auth/Proxy. mainConfig := suite.defaultServiceConfig() @@ -7537,6 +7540,53 @@ func testJoinOverReverseTunnelOnly(t *testing.T, suite *integrationTestSuite) { require.NoError(t, err, "Node failed to join over reverse tunnel") }) } + + // Assert that gRPC-based join methods work over reverse tunnel. + t.Run("gRPC join service", func(t *testing.T) { + lib.SetInsecureDevMode(true) + defer lib.SetInsecureDevMode(false) + + // Create a Teleport instance with Auth/Proxy. + mainConfig := suite.defaultServiceConfig() + mainConfig.Auth.Enabled = true + + mainConfig.Proxy.Enabled = true + mainConfig.Proxy.DisableWebService = false + mainConfig.Proxy.DisableWebInterface = true + + mainConfig.SSH.Enabled = false + + main := suite.NewTeleportWithConfig(t, nil, nil, mainConfig) + t.Cleanup(func() { require.NoError(t, main.StopAll()) }) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + dialer := apiclient.NewDialer( + ctx, + apidefaults.DefaultIdleTimeout, + apidefaults.DefaultIOTimeout, + ) + tlsConfig := utils.TLSConfig(nil) + tlsConfig.InsecureSkipVerify = true + tlsConfig.NextProtos = []string{string(common.ProtocolProxyGRPCInsecure)} + conn, err := grpc.Dial( + main.ReverseTunnel, + grpc.WithContextDialer(apiclient.GRPCContextDialer(dialer)), + grpc.WithUnaryInterceptor(metadata.UnaryClientInterceptor), + grpc.WithStreamInterceptor(metadata.StreamClientInterceptor), + grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), + ) + require.NoError(t, err) + joinServiceClient := apiclient.NewJoinServiceClient(proto.NewJoinServiceClient(conn)) + _, err = joinServiceClient.RegisterUsingAzureMethod(ctx, func(challenge string) (*proto.RegisterUsingAzureMethodRequest, error) { + return &proto.RegisterUsingAzureMethodRequest{ + RegisterUsingTokenRequest: &types.RegisterUsingTokenRequest{}, + }, nil + }) + // We don't care about the join succeeding, we just want to confirm + // that gRPC works. + require.ErrorContains(t, err, "missing parameter AttestedData") + }) } func getRemoteAddrString(sshClientString string) string { diff --git a/lib/service/service.go b/lib/service/service.go index 492e822546cba..486409f60191b 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -3225,16 +3225,23 @@ type proxyListeners struct { kube net.Listener db dbListeners alpn net.Listener - proxyPeer net.Listener + // reverseTunnelALPN handles ALPN traffic on the reverse tunnel port when TLS routing + // is not enabled. It's used to redirect traffic on that port to the gRPC + // listener. + reverseTunnelALPN net.Listener + proxyPeer net.Listener // grpcPublic receives gRPC traffic that has the TLS ALPN protocol common.ProtocolProxyGRPCInsecure. This - // listener is only enabled when TLS routing is enabled and does not enforce mTLS authentication since - // it's used to handle cluster join requests. + // listener does not enforce mTLS authentication since it's used to handle cluster join requests. grpcPublic net.Listener // grpcMTLS receives gRPC traffic that has the TLS ALPN protocol common.ProtocolProxyGRPCSecure. This // listener is only enabled when TLS routing is enabled and the gRPC server will enforce mTLS authentication. grpcMTLS net.Listener reverseTunnelMux *multiplexer.Mux - minimalTLS *multiplexer.WebListener + // minimalWeb handles traffic on the reverse tunnel port when TLS routing + // is not enabled. It serves only the subset of web traffic required for + // agents to join the cluster. + minimalWeb net.Listener + minimalTLS *multiplexer.WebListener } // Close closes all proxy listeners. @@ -3267,6 +3274,9 @@ func (l *proxyListeners) Close() { if l.alpn != nil { l.alpn.Close() } + if l.reverseTunnelALPN != nil { + l.reverseTunnelALPN.Close() + } if l.proxyPeer != nil { l.proxyPeer.Close() } @@ -3279,6 +3289,9 @@ func (l *proxyListeners) Close() { if l.reverseTunnelMux != nil { l.reverseTunnelMux.Close() } + if l.minimalWeb != nil { + l.minimalWeb.Close() + } if l.minimalTLS != nil { l.minimalTLS.Close() } @@ -3570,6 +3583,7 @@ func (process *TeleportProcess) initMinimalReverseTunnelListener(cfg *servicecfg process.log.WithError(err).Debug("Minimal reverse tunnel mux exited with error") } }() + listeners.minimalWeb = listeners.reverseTunnelMux.TLS() return nil } @@ -3701,7 +3715,7 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error { if err != nil { return trace.Wrap(err) } - alpnRouter := setupALPNRouter(listeners, serverTLSConfig, cfg) + alpnRouter, reverseTunnelALPNRouter := setupALPNRouter(listeners, serverTLSConfig, cfg) alpnAddr := "" if listeners.alpn != nil { @@ -4424,6 +4438,7 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error { } var alpnServer *alpnproxy.Proxy + var reverseTunnelALPNServer *alpnproxy.Proxy if !cfg.Proxy.DisableTLS && !cfg.Proxy.DisableALPNSNIListener && listeners.web != nil { authDialerService := alpnproxyauth.NewAuthProxyDialerService( tsrv, @@ -4464,6 +4479,28 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error { } return nil }) + + if reverseTunnelALPNRouter != nil { + reverseTunnelALPNServer, err = alpnproxy.New(alpnproxy.ProxyConfig{ + WebTLSConfig: tlsConfigWeb.Clone(), + IdentityTLSConfig: identityTLSConf, + Router: reverseTunnelALPNRouter, + Listener: listeners.reverseTunnelALPN, + ClusterName: clusterName, + AccessPoint: accessPoint, + }) + if err != nil { + return trace.Wrap(err) + } + + process.RegisterCriticalFunc("proxy.tls.alpn.sni.proxy.reverseTunnel", func() error { + log.Infof("Starting TLS ALPN SNI reverse tunnel proxy server on %v.", listeners.reverseTunnelALPN.Addr()) + if err := reverseTunnelALPNServer.Serve(process.ExitContext()); err != nil { + log.WithError(err).Warn("TLS ALPN SNI proxy proxy on reverse tunnel server exited with error.") + } + return nil + }) + } } // execute this when process is asked to exit: @@ -4504,6 +4541,9 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error { if alpnServer != nil { warnOnErr(alpnServer.Close(), log) } + if reverseTunnelALPNServer != nil { + warnOnErr(reverseTunnelALPNServer.Close(), log) + } } else { log.Infof("Shutting down gracefully.") ctx := payloadContext(payload, log) @@ -4539,6 +4579,9 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error { if alpnServer != nil { warnOnErr(alpnServer.Close(), log) } + if reverseTunnelALPNServer != nil { + warnOnErr(reverseTunnelALPNServer.Close(), log) + } // Explicitly deleting proxy heartbeats helps the behavior of // reverse tunnel agents during rollouts, as otherwise they'll keep @@ -4580,7 +4623,7 @@ func (process *TeleportProcess) getPROXYSigner(ident *auth.Identity) (multiplexe } func (process *TeleportProcess) initMinimalReverseTunnel(listeners *proxyListeners, tlsConfigWeb *tls.Config, cfg *servicecfg.Config, webConfig web.Config, log *logrus.Entry) (*web.Server, error) { - internalListener := listeners.reverseTunnelMux.TLS() + internalListener := listeners.minimalWeb if !cfg.Proxy.DisableTLS { internalListener = tls.NewListener(internalListener, tlsConfigWeb) } @@ -4752,15 +4795,20 @@ func (process *TeleportProcess) setupALPNTLSConfigForWeb(serverTLSConfig *tls.Co return tlsConfig } -func setupALPNRouter(listeners *proxyListeners, serverTLSConfig *tls.Config, cfg *servicecfg.Config) *alpnproxy.Router { +func setupALPNRouter(listeners *proxyListeners, serverTLSConfig *tls.Config, cfg *servicecfg.Config) (router, rtRouter *alpnproxy.Router) { if listeners.web == nil || cfg.Proxy.DisableTLS || cfg.Proxy.DisableALPNSNIListener { - return nil + return nil, nil } // ALPN proxy service will use web listener where listener.web will be overwritten by alpn wrapper // that allows to dispatch the http/1.1 and h2 traffic to webService. listeners.alpn = listeners.web + router = alpnproxy.NewRouter() + + if listeners.minimalWeb != nil { + listeners.reverseTunnelALPN = listeners.minimalWeb + rtRouter = alpnproxy.NewRouter() + } - router := alpnproxy.NewRouter() if cfg.Proxy.Kube.Enabled { kubeListener := alpnproxy.NewMuxListenerWrapper(listeners.kube, listeners.web) router.AddKubeHandler(kubeListener.HandleConnection) @@ -4773,6 +4821,21 @@ func setupALPNRouter(listeners *proxyListeners, serverTLSConfig *tls.Config, cfg Handler: reverseTunnel.HandleConnection, }) listeners.reverseTunnel = reverseTunnel + + if rtRouter != nil { + minimalWeb := alpnproxy.NewMuxListenerWrapper(nil, listeners.reverseTunnelALPN) + rtRouter.Add(alpnproxy.HandlerDecs{ + MatchFunc: alpnproxy.MatchByProtocol( + alpncommon.ProtocolHTTP, + alpncommon.ProtocolHTTP2, + alpncommon.ProtocolDefault, + ), + Handler: minimalWeb.HandleConnection, + ForwardTLS: true, + }) + listeners.minimalWeb = minimalWeb + } + } if !cfg.Proxy.DisableWebService { @@ -4792,10 +4855,17 @@ func setupALPNRouter(listeners *proxyListeners, serverTLSConfig *tls.Config, cfg // It must not be used for any services that require authentication and currently // it is only used by the join service which nodes rely on to join the cluster. grpcPublicListener := alpnproxy.NewMuxListenerWrapper(nil /* serviceListener */, listeners.web) + grpcPublicListener = alpnproxy.NewMuxListenerWrapper(grpcPublicListener, listeners.reverseTunnel) router.Add(alpnproxy.HandlerDecs{ MatchFunc: alpnproxy.MatchByProtocol(alpncommon.ProtocolProxyGRPCInsecure), Handler: grpcPublicListener.HandleConnection, }) + if rtRouter != nil { + rtRouter.Add(alpnproxy.HandlerDecs{ + MatchFunc: alpnproxy.MatchByProtocol(alpncommon.ProtocolProxyGRPCInsecure), + Handler: grpcPublicListener.HandleConnection, + }) + } listeners.grpcPublic = grpcPublicListener // grpcSecureListener is a listener that is used by a gRPC server that enforces @@ -4831,7 +4901,7 @@ func setupALPNRouter(listeners *proxyListeners, serverTLSConfig *tls.Config, cfg router.AddDBTLSHandler(webTLSDB.HandleConnection) listeners.db.tls = webTLSDB - return router + return router, rtRouter } // waitForAppDepend waits until all dependencies for an application service