service.go

/*
Copyright 2015-2017 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package service implements teleport running service, takes care
// of initialization, cleanup and shutdown procedures
package service

import (
	"context"
	"crypto/tls"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"

	"golang.org/x/crypto/ssh"

	"github.com/gravitational/teleport"
	"github.com/gravitational/teleport/lib/auth"
	"github.com/gravitational/teleport/lib/auth/native"
	"github.com/gravitational/teleport/lib/backend"
	"github.com/gravitational/teleport/lib/backend/boltbk"
	"github.com/gravitational/teleport/lib/backend/dir"
	"github.com/gravitational/teleport/lib/backend/dynamo"
	"github.com/gravitational/teleport/lib/backend/etcdbk"
	"github.com/gravitational/teleport/lib/defaults"
	"github.com/gravitational/teleport/lib/events"
	"github.com/gravitational/teleport/lib/events/dynamoevents"
	"github.com/gravitational/teleport/lib/events/filesessions"
	"github.com/gravitational/teleport/lib/events/s3sessions"
	"github.com/gravitational/teleport/lib/limiter"
	"github.com/gravitational/teleport/lib/multiplexer"
	"github.com/gravitational/teleport/lib/reversetunnel"
	"github.com/gravitational/teleport/lib/services"
	"github.com/gravitational/teleport/lib/session"
	"github.com/gravitational/teleport/lib/srv/regular"
	"github.com/gravitational/teleport/lib/state"
	"github.com/gravitational/teleport/lib/system"
	"github.com/gravitational/teleport/lib/utils"
	"github.com/gravitational/teleport/lib/web"
	"github.com/gravitational/trace"

	"github.com/gravitational/roundtrip"
	"github.com/jonboulle/clockwork"
	"github.com/pborman/uuid"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/sirupsen/logrus"
)

var log = logrus.WithFields(logrus.Fields{
	trace.Component: teleport.ComponentProcess,
})

const (
	// AuthIdentityEvent is generated when the Auth Servers identity has been
	// initialized in the backend.
	AuthIdentityEvent = "AuthIdentity"

	// ProxyIdentityEvent is generated by the supervisor when the proxy's
	// identity has been registered with the Auth Server.
	ProxyIdentityEvent = "ProxyIdentity"

	// SSHIdentityEvent is generated when node's identity has been registered
	// with the Auth Server.
	SSHIdentityEvent = "SSHIdentity"

	// DELETE IN: 2.6.0
	// AuthSSHReady is generated when the Auth Server has initialized the
	// HTTP-over-SSH endpoint and is ready to start accepting connections.
	AuthSSHReady = "AuthSSHReady"

	// AuthTLSReady is generated when the Auth Server has initialized the
	// TLS Mutual Auth endpoint and is ready to start accepting connections.
	AuthTLSReady = "AuthTLSReady"

	// ProxyWebServerReady is generated when the proxy has initialized the web
	// server and is ready to start accepting connections.
	ProxyWebServerReady = "ProxyWebServerReady"

	// ProxyReverseTunnelReady is generated when the proxy has initialized the
	// reverse tunnel server and is ready to start accepting connections.
	ProxyReverseTunnelReady = "ProxyReverseTunnelReady"

	// ProxySSHReady is generated when the proxy has initialized a SSH server
	// and is ready to start accepting connections.
	ProxySSHReady = "ProxySSHReady"

	// NodeReady is generated when the Teleport node has initialized a SSH server
	// and is ready to start accepting SSH connections.
	NodeSSHReady = "NodeReady"

	// TeleportExitEvent is generated when the Teleport process begins closing
	// all listening sockets and exiting.
	TeleportExitEvent = "TeleportExit"
)

// RoleConfig is a configuration for a server role (either proxy or node)
type RoleConfig struct {
	DataDir     string
	HostUUID    string
	HostName    string
	AuthServers []utils.NetAddr
	Auth        AuthConfig
	Console     io.Writer
}

// Connector has all resources process needs to connect
// to other parts of the cluster: client and identity
type Connector struct {
	Identity *auth.Identity
	Client   *auth.Client
}

// TeleportProcess structure holds the state of the Teleport daemon, controlling
// execution and configuration of the teleport services: ssh, auth and proxy.
type TeleportProcess struct {
	clockwork.Clock
	sync.Mutex
	Supervisor
	Config *Config
	// localAuth has local auth server listed in case if this process
	// has started with auth server role enabled
	localAuth *auth.AuthServer
	// backend is the process' backend
	backend backend.Backend
	// auditLog is the initialized audit log
	auditLog events.IAuditLog

	// identities of this process (credentials to auth sever, basically)
	Identities map[teleport.Role]*auth.Identity
	// registeredListeners keeps track of all listeners created by the process
	// used to pass listeners to child processes during live reload
	registeredListeners []RegisteredListener
	// importedDescriptors is a list of imported file descriptors
	// passed by the parent process
	importedDescriptors []FileDescriptor
}

// GetAuthServer returns the process' auth server
func (process *TeleportProcess) GetAuthServer() *auth.AuthServer {
	return process.localAuth
}

// GetAuditLog returns the process' audit log
func (process *TeleportProcess) GetAuditLog() events.IAuditLog {
	return process.auditLog
}

// GetBackend returns the process' backend
func (process *TeleportProcess) GetBackend() backend.Backend {
	return process.backend
}

func (process *TeleportProcess) backendSupportsForks() bool {
	switch process.backend.(type) {
	case *boltbk.BoltBackend:
		return false
	default:
		return true
	}
}

func (process *TeleportProcess) findStaticIdentity(id auth.IdentityID) (*auth.Identity, error) {
	for i := range process.Config.Identities {
		identity := process.Config.Identities[i]
		if identity.ID.Equals(id) {
			return identity, nil
		}
	}
	return nil, trace.NotFound("identity %v not found", &id)
}

// readIdentity reads identity from disk and resets the local state
func (process *TeleportProcess) readIdentity(role teleport.Role) (*auth.Identity, error) {
	process.Lock()
	defer process.Unlock()

	id := auth.IdentityID{HostUUID: process.Config.HostUUID, Role: role}
	identity, err := auth.ReadIdentity(process.Config.DataDir, id)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	process.Identities[role] = identity
	return identity, nil
}

// GetIdentity returns the process identity (credentials to the auth server) for a given
// teleport Role. A teleport process can have any combination of 3 roles: auth, node, proxy
// and they have their own identities
func (process *TeleportProcess) GetIdentity(role teleport.Role) (i *auth.Identity, err error) {
	var found bool

	process.Lock()
	defer process.Unlock()

	i, found = process.Identities[role]
	if found {
		return i, nil
	}

	id := auth.IdentityID{HostUUID: process.Config.HostUUID, Role: role}
	i, err = auth.ReadIdentity(process.Config.DataDir, id)
	if err != nil {
		if trace.IsNotFound(err) {
			// try to locate static identity provided in the file
			i, err = process.findStaticIdentity(id)
			if err != nil {
				return nil, trace.Wrap(err)
			}
			log.Infof("Found static identity %v in the config file, writing to disk.", &id)
			if err = auth.WriteIdentity(process.Config.DataDir, i); err != nil {
				return nil, trace.Wrap(err)
			}
		} else {
			return nil, trace.Wrap(err)
		}
	}
	process.Identities[role] = i
	return i, nil
}

// connectToAuthService attempts to login into the auth servers specified in the
// configuration. Returns 'true' if successful
func (process *TeleportProcess) connectToAuthService(role teleport.Role, additionalPrincipals []string) (*Connector, error) {
	identity, err := process.GetIdentity(role)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	tlsConfig, err := identity.TLSConfig()
	if err != nil {
		if !trace.IsNotFound(err) {
			return nil, trace.Wrap(err)
		}
		// connect using legacy SSH and get new set of TLS credentials
		storage := utils.NewFileAddrStorage(
			filepath.Join(process.Config.DataDir, "authservers.json"))

		authUser := identity.Cert.ValidPrincipals[0]
		log.Infof("Connecting to the cluster as %v to fetch TLS certificates.", authUser)
		authClient, err := auth.NewTunClient(
			string(role),
			process.Config.AuthServers,
			authUser,
			[]ssh.AuthMethod{ssh.PublicKeys(identity.KeySigner)},
			auth.TunClientStorage(storage),
			auth.TunDisableRefresh(),
		)
		if err != nil {
			return nil, trace.Wrap(err)
		}
		defer authClient.Close()
		if err := auth.ReRegister(process.Config.DataDir, authClient, identity.ID, additionalPrincipals); err != nil {
			return nil, trace.Wrap(err)
		}
		if identity, err = process.readIdentity(role); err != nil {
			return nil, trace.Wrap(err)
		}
		tlsConfig, err = identity.TLSConfig()
		if err != nil {
			return nil, trace.Wrap(err)
		}
		log.WithFields(logrus.Fields{"host": identity.ID.HostUUID, "role": identity.ID.Role}).Infof("Received new TLS identity.")
	}
	log.Infof("Connecting to the cluster %v with TLS client certificate.", identity.ClusterName)
	client, err := auth.NewTLSClient(process.Config.AuthServers, tlsConfig)
	if err != nil {
		return nil, trace.Wrap(err)
	}
	if len(additionalPrincipals) != 0 && !identity.HasPrincipals(additionalPrincipals) {
		log.Infof("Identity %v needs principals %v, going to re-register.", identity.ID, additionalPrincipals)
		if err := auth.ReRegister(process.Config.DataDir, client, identity.ID, additionalPrincipals); err != nil {
			return nil, trace.Wrap(err)
		}
		if identity, err = process.readIdentity(role); err != nil {
			return nil, trace.Wrap(err)
		}
		tlsConfig, err = identity.TLSConfig()
		if err != nil {
			return nil, trace.Wrap(err)
		}
	}
	// success ? we're logged in!
	return &Connector{Client: client, Identity: identity}, nil
}

// NewTeleport takes the daemon configuration, instantiates all required services
// and starts them under a supervisor, returning the supervisor object
func NewTeleport(cfg *Config) (*TeleportProcess, error) {
	// before we do anything reset the SIGINT handler back to the default
	system.ResetInterruptSignalHandler()

	if err := validateConfig(cfg); err != nil {
		return nil, trace.Wrap(err, "configuration error")
	}

	// create the data directory if it's missing
	_, err := os.Stat(cfg.DataDir)
	if os.IsNotExist(err) {
		err := os.MkdirAll(cfg.DataDir, os.ModeDir|0700)
		if err != nil {
			return nil, trace.Wrap(err)
		}
	}

	// create the pid file
	if cfg.PIDFile != "" {
		f, err := os.OpenFile(cfg.PIDFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
		if err != nil {
			return nil, trace.ConvertSystemError(err)
		}
		fmt.Fprintf(f, "%v", os.Getpid())
		defer f.Close()
	}

	importedDescriptors, err := importFileDescriptors()
	if err != nil {
		return nil, trace.Wrap(err)
	}

	// if there's no host uuid initialized yet, try to read one from the
	// one of the identities
	cfg.HostUUID, err = utils.ReadHostUUID(cfg.DataDir)
	if err != nil {
		if !trace.IsNotFound(err) {
			return nil, trace.Wrap(err)
		}
		if len(cfg.Identities) != 0 {
			cfg.HostUUID = cfg.Identities[0].ID.HostUUID
			log.Infof("Taking host UUID from first identity: %v.", cfg.HostUUID)
		} else {
			cfg.HostUUID = uuid.New()
			log.Infof("Generating new host UUID: %v.", cfg.HostUUID)
		}
		if err := utils.WriteHostUUID(cfg.DataDir, cfg.HostUUID); err != nil {
			return nil, trace.Wrap(err)
		}
	}

	// if user started auth and another service (without providing the auth address for
	// that service, the address of the in-process auth will be used
	if cfg.Auth.Enabled && len(cfg.AuthServers) == 0 {
		cfg.AuthServers = []utils.NetAddr{cfg.Auth.SSHAddr}
	}

	// if user did not provide auth domain name, use this host's name
	if cfg.Auth.Enabled && cfg.Auth.ClusterName == nil {
		cfg.Auth.ClusterName, err = services.NewClusterName(services.ClusterNameSpecV2{
			ClusterName: cfg.Hostname,
		})
		if err != nil {
			return nil, trace.Wrap(err)
		}
	}

	process := &TeleportProcess{
		Clock:               clockwork.NewRealClock(),
		Supervisor:          NewSupervisor(),
		Config:              cfg,
		Identities:          make(map[teleport.Role]*auth.Identity),
		importedDescriptors: importedDescriptors,
	}

	serviceStarted := false

	if !cfg.DiagnosticAddr.IsEmpty() {
		if err := process.initDiagnosticService(); err != nil {
			return nil, trace.Wrap(err)
		}
	} else {
		warnOnErr(process.closeImportedDescriptors(teleport.ComponentDiagnostic))
	}

	// Create a process wide key generator that will be shared. This is so the
	// key generator can pre-generate keys and share these across services.
	if cfg.Keygen == nil {
		cfg.Keygen = native.New()
	}

	if cfg.Auth.Enabled {
		if err := process.initAuthService(); err != nil {
			return nil, trace.Wrap(err)
		}
		serviceStarted = true
	} else {
		warnOnErr(process.closeImportedDescriptors(teleport.ComponentAuth))
	}

	if cfg.SSH.Enabled {
		if err := process.initSSH(); err != nil {
			return nil, err
		}
		serviceStarted = true
	} else {
		warnOnErr(process.closeImportedDescriptors(teleport.ComponentNode))
	}

	if cfg.Proxy.Enabled {
		if err := process.initProxy(); err != nil {
			return nil, err
		}
		serviceStarted = true
	} else {
		warnOnErr(process.closeImportedDescriptors(teleport.ComponentProxy))
	}

	if !serviceStarted {
		return nil, trace.BadParameter("all services failed to start")
	}

	return process, nil
}

func (process *TeleportProcess) setLocalAuth(a *auth.AuthServer) {
	process.Lock()
	defer process.Unlock()
	process.localAuth = a
}

func (process *TeleportProcess) getLocalAuth() *auth.AuthServer {
	process.Lock()
	defer process.Unlock()
	return process.localAuth
}

// adminCreds returns admin UID and GID settings based on the OS
func adminCreds() (*int, *int, error) {
	if runtime.GOOS != teleport.LinuxOS {
		return nil, nil, nil
	}
	// if the user member of adm linux group,
	// make audit log folder readable by admins
	isAdmin, err := utils.IsGroupMember(teleport.LinuxAdminGID)
	if err != nil {
		return nil, nil, trace.Wrap(err)
	}
	if !isAdmin {
		return nil, nil, nil
	}
	uid := os.Getuid()
	gid := teleport.LinuxAdminGID
	return &uid, &gid, nil
}

// initUploadHandler initializes upload handler based on the config settings,
// currently the only upload handler supported is S3
// the call can return trace.NotFOund if no upload handler is setup
func initUploadHandler(auditConfig services.AuditConfig) (events.UploadHandler, error) {
	if auditConfig.AuditSessionsURI == "" {
		return nil, trace.NotFound("no upload handler is setup")
	}
	uri, err := utils.ParseSessionsURI(auditConfig.AuditSessionsURI)
	if err != nil {
		return nil, trace.Wrap(err)
	}

	switch uri.Scheme {
	case teleport.SchemeS3:
		return s3sessions.NewHandler(s3sessions.Config{
			Bucket: uri.Host,
			Region: auditConfig.Region,
		})
	case teleport.SchemeFile:
		return filesessions.NewHandler(filesessions.Config{
			Directory: uri.Path,
		})
	default:
		return nil, trace.BadParameter(
			"unsupported scheme for audit_sesions_uri: %q, currently supported schemes are %q and %q",
			uri.Scheme, teleport.SchemeS3)
	}
}

// initExternalLog initializes external storage, if the storage is not
// setup, returns nil
func initExternalLog(auditConfig services.AuditConfig) (events.IAuditLog, error) {
	if auditConfig.Type == "" {
		if auditConfig.AuditTableName != "" {
			return nil, trace.BadParameter("no storage type defined for table name %q in config %#v", auditConfig.AuditTableName, auditConfig)
		}
		return nil, trace.NotFound("no external log is defined")
	}
	if auditConfig.AuditTableName == "" {
		return nil, trace.NotFound("no external log is defined")
	}
	if auditConfig.Type != dynamo.GetName() {
		return nil, trace.BadParameter("unsupported events backend %q, the only supported backend is %q", auditConfig.Type, dynamo.GetName())
	}
	if !auditConfig.ShouldUploadSessions() {
		return nil, trace.BadParameter("please specify audit_sessions_uri when using external audit backends")
	}
	return dynamoevents.New(dynamoevents.Config{
		Tablename: auditConfig.AuditTableName,
		Region:    auditConfig.Region,
	})
}

// initAuthService can be called to initialize auth server service
func (process *TeleportProcess) initAuthService() error {
	var (
		askedToExit = false
		err         error
	)
	cfg := process.Config

	// Initialize the storage back-ends for keys, events and records
	b, err := process.initAuthStorage()
	if err != nil {
		return trace.Wrap(err)
	}
	process.backend = b

	// create the audit log, which will be consuming (and recording) all events
	// and recording all sessions.
	if cfg.Auth.NoAudit {
		// this is for teleconsole
		process.auditLog = events.NewDiscardAuditLog()

		warningMessage := "Warning: Teleport audit and session recording have been " +
			"turned off. This is dangerous, you will not be able to view audit events " +
			"or save and playback recorded sessions."
		log.Warn(warningMessage)
	} else {
		// check if session recording has been disabled. note, we will continue
		// logging audit events, we just won't record sessions.
		recordSessions := true
		if cfg.Auth.ClusterConfig.GetSessionRecording() == services.RecordOff {
			recordSessions = false

			warningMessage := "Warning: Teleport session recording have been turned off. " +
				"This is dangerous, you will not be able to save and playback sessions."
			log.Warn(warningMessage)
		}

		auditConfig := cfg.Auth.ClusterConfig.GetAuditConfig()
		uploadHandler, err := initUploadHandler(auditConfig)
		if err != nil {
			if !trace.IsNotFound(err) {
				return trace.Wrap(err)
			}
		}

		externalLog, err := initExternalLog(auditConfig)
		if err != nil {
			if !trace.IsNotFound(err) {
				return trace.Wrap(err)
			}
		}

		auditServiceConfig := events.AuditLogConfig{
			DataDir:        filepath.Join(cfg.DataDir, teleport.LogsDir),
			RecordSessions: recordSessions,
			ServerID:       cfg.HostUUID,
			UploadHandler:  uploadHandler,
			ExternalLog:    externalLog,
		}
		auditServiceConfig.UID, auditServiceConfig.GID, err = adminCreds()
		if err != nil {
			return trace.Wrap(err)
		}
		process.auditLog, err = events.NewAuditLog(auditServiceConfig)
		if err != nil {
			return trace.Wrap(err)
		}
	}

	// first, create the AuthServer
	authServer, identity, err := auth.Init(auth.InitConfig{
		Backend:              b,
		Authority:            cfg.Keygen,
		ClusterConfiguration: cfg.ClusterConfiguration,
		ClusterConfig:        cfg.Auth.ClusterConfig,
		ClusterName:          cfg.Auth.ClusterName,
		AuthServiceName:      cfg.Hostname,
		DataDir:              cfg.DataDir,
		HostUUID:             cfg.HostUUID,
		NodeName:             cfg.Hostname,
		Authorities:          cfg.Auth.Authorities,
		ReverseTunnels:       cfg.ReverseTunnels,
		Trust:                cfg.Trust,
		Presence:             cfg.Presence,
		Provisioner:          cfg.Provisioner,
		Identity:             cfg.Identity,
		Access:               cfg.Access,
		StaticTokens:         cfg.Auth.StaticTokens,
		Roles:                cfg.Auth.Roles,
		AuthPreference:       cfg.Auth.Preference,
		OIDCConnectors:       cfg.OIDCConnectors,
		AuditLog:             process.auditLog,
	})
	if err != nil {
		return trace.Wrap(err)
	}

	process.setLocalAuth(authServer)

	// second, create the API Server: it's actually a collection of API servers,
	// each serving requests for a "role" which is assigned to every connected
	// client based on their certificate (user, server, admin, etc)
	sessionService, err := session.New(b)
	if err != nil {
		return trace.Wrap(err)
	}
	authorizer, err := auth.NewAuthorizer(authServer.Access, authServer.Identity, authServer.Trust)
	if err != nil {
		return trace.Wrap(err)
	}
	apiConf := &auth.APIConfig{
		AuthServer:     authServer,
		SessionService: sessionService,
		Authorizer:     authorizer,
		AuditLog:       process.auditLog,
	}

	sshLimiter, err := limiter.NewLimiter(cfg.Auth.Limiter)
	if err != nil {
		return trace.Wrap(err)
	}

	// admin access point is a caching access point used for frequently
	// accessed data by auth server, e.g. cert authorities, users and roles
	adminAuthServer, err := auth.NewAdminAuthServer(authServer, sessionService, process.auditLog)
	if err != nil {
		return trace.Wrap(err)
	}
	adminAccessPoint, err := process.newLocalCache(adminAuthServer, []string{"auth"})
	if err != nil {
		return trace.Wrap(err)
	}

	log := logrus.WithFields(logrus.Fields{
		trace.Component: teleport.ComponentAuth,
	})

	// Register TLS endpoint of the auth service
	tlsConfig, err := identity.TLSConfig()
	if err != nil {
		return trace.Wrap(err)
	}
	tlsServer, err := auth.NewTLSServer(auth.TLSServerConfig{
		TLS:           tlsConfig,
		APIConfig:     *apiConf,
		LimiterConfig: cfg.Auth.Limiter,
		AccessPoint:   adminAccessPoint,
	})
	if err != nil {
		return trace.Wrap(err)
	}

	// auth server listens on SSH and TLS, reusing the same socket
	listener, err := process.importOrCreateListener(teleport.ComponentAuth, cfg.Auth.SSHAddr.Addr)
	if err != nil {
		log.Errorf("PID: %v Failed to bind to address %v: %v, exiting.", os.Getpid(), cfg.Auth.SSHAddr.Addr, err)
		return trace.Wrap(err)
	}
	// clean up unused descriptors passed for proxy, but not used by it
	warnOnErr(process.closeImportedDescriptors(teleport.ComponentAuth))
	if cfg.Auth.EnableProxyProtocol {
		log.Infof("Starting Auth service with PROXY protocol support.")
	}
	mux, err := multiplexer.New(multiplexer.Config{
		EnableProxyProtocol: cfg.Auth.EnableProxyProtocol,
		Listener:            listener,
	})
	if err != nil {
		listener.Close()
		return trace.Wrap(err)
	}
	go mux.Serve()

	// Register an SSH endpoint which is used to create an SSH tunnel to send HTTP
	// requests to the Auth API
	var authTunnel *auth.AuthTunnel
	process.RegisterFunc("auth.ssh", func() error {
		log.Infof("Auth SSH service is starting on %v", cfg.Auth.SSHAddr.Addr)
		authTunnel, err = auth.NewTunnel(
			cfg.Auth.SSHAddr,
			identity.KeySigner,
			apiConf,
			auth.SetLimiter(sshLimiter),
		)
		if err != nil {
			log.Errorf("Error: %v", err)
			return trace.Wrap(err)
		}

		// since authTunnel.Serve is a blocking call, we emit this even right before
		// the service has started
		process.BroadcastEvent(Event{Name: AuthSSHReady, Payload: nil})

		if err := authTunnel.Serve(mux.SSH()); err != nil {
			if askedToExit {
				log.Infof("Auth tunnel exited.")
				return nil
			}
			log.Errorf("Error: %v", err)
			return trace.Wrap(err)
		}
		return nil
	})

	process.RegisterFunc("auth.tls", func() error {
		// since tlsServer.Serve is a blocking call, we emit this even right before
		// the service has started
		process.BroadcastEvent(Event{Name: AuthTLSReady, Payload: nil})

		err := tlsServer.Serve(mux.TLS())
		if err != nil && err != http.ErrServerClosed {
			log.Warningf("TLS server exited with error: %v.", err)
		}
		return nil
	})

	process.RegisterFunc("auth.heartbeat.broadcast", func() error {
		// Heart beat auth server presence, this is not the best place for this
		// logic, consolidate it into auth package later
		connector, err := process.connectToAuthService(teleport.RoleAdmin, nil)
		if err != nil {
			return trace.Wrap(err)
		}
		// External integrations rely on this event:
		process.BroadcastEvent(Event{Name: AuthIdentityEvent, Payload: connector})
		process.onExit("auth.broadcast", func(payload interface{}) {
			connector.Client.Close()
		})
		return nil
	})

	closeContext, signalClose := context.WithCancel(context.TODO())

	process.RegisterFunc("auth.heartbeat", func() error {
		srv := services.ServerV2{
			Kind:    services.KindAuthServer,
			Version: services.V2,
			Metadata: services.Metadata{
				Namespace: defaults.Namespace,
				Name:      process.Config.HostUUID,
			},
			Spec: services.ServerSpecV2{
				Addr:     cfg.Auth.SSHAddr.Addr,
				Hostname: process.Config.Hostname,
			},
		}
		host, port, err := net.SplitHostPort(srv.GetAddr())
		// advertise-ip is explicitly set:
		if process.Config.AdvertiseIP != nil {
			if err != nil {
				return trace.Wrap(err)
			}
			srv.SetAddr(fmt.Sprintf("%v:%v", process.Config.AdvertiseIP.String(), port))
		} else {
			// advertise-ip is not set, while the CA is listening on 0.0.0.0? lets try
			// to guess the 'advertise ip' then:
			if net.ParseIP(host).IsUnspecified() {
				ip, err := utils.GuessHostIP()
				if err != nil {
					log.Warn(err)
				} else {
					srv.SetAddr(net.JoinHostPort(ip.String(), port))
				}
			}
			log.Warnf("auth_service: advertise_ip is not set. guessing %v.", srv.GetAddr())
		}
		// immediately register, and then keep repeating in a loop:
		ticker := time.NewTicker(defaults.ServerHeartbeatTTL / 2)
		defer ticker.Stop()
	announce:
		for {
			srv.SetTTL(process, defaults.ServerHeartbeatTTL)
			err := authServer.UpsertAuthServer(&srv)
			if err != nil {
				log.Warningf("Failed to announce presence: %v.", err)
			}
			select {
			case <-closeContext.Done():
				break announce
			case <-ticker.C:
			}
		}
		log.Infof("Heartbeat to other auth servers exited.")
		return nil
	})

	// execute this when process is asked to exit:
	process.onExit("auth.shutdown", func(payload interface{}) {
		// as a last resort, at least close listeners (e.g. panic)
		if listener != nil {
			defer listener.Close()
		}
		if mux != nil {
			defer mux.Close()
		}
		signalClose()
		if payload == nil {
			log.Info("Shutting down immediately.")
			warnOnErr(tlsServer.Close())
			warnOnErr(authTunnel.Close())
		} else {
			log.Info("Shutting down gracefully.")
			ctx := payloadContext(payload)
			warnOnErr(tlsServer.Shutdown(ctx))
			warnOnErr(authTunnel.Shutdown(ctx))
		}
		log.Info("Exited.")
	})
	return nil
}

func payloadContext(payload interface{}) context.Context {
	ctx, ok := payload.(context.Context)
	if ok {
		return ctx
	}
	log.Errorf("expected context, got %T", payload)
	return context.TODO()
}

// onExit allows individual services to register a callback function which will be
// called when Teleport Process is asked to exit. Usually services terminate themselves
// when the callback is called
func (process *TeleportProcess) onExit(serviceName string, callback func(interface{})) {
	process.RegisterFunc(serviceName, func() error {
		eventC := make(chan Event)
		process.WaitForEvent(TeleportExitEvent, eventC, make(chan struct{}))
		select {
		case event := <-eventC:
			callback(event.Payload)
		}
		return nil
	})
}

// newLocalCache returns new local cache access point
func (process *TeleportProcess) newLocalCache(clt auth.ClientI, cacheName []string) (auth.AccessPoint, error) {
	// if caching is disabled, return access point
	if !process.Config.CachePolicy.Enabled {
		return clt, nil
	}
	path := filepath.Join(append([]string{process.Config.DataDir, "cache"}, cacheName...)...)
	if err := os.MkdirAll(path, teleport.SharedDirMode); err != nil {
		return nil, trace.ConvertSystemError(err)
	}
	cacheBackend, err := dir.New(backend.Params{"path": path})
	if err != nil {
		return nil, trace.Wrap(err)
	}
	return state.NewCachingAuthClient(state.Config{
		AccessPoint:    clt,
		Backend:        cacheBackend,
		NeverExpires:   process.Config.CachePolicy.NeverExpires,
		RecentCacheTTL: process.Config.CachePolicy.GetRecentTTL(),
		CacheMaxTTL:    process.Config.CachePolicy.TTL,
	})
}

// initSSH initializes the "node" role, i.e. a simple SSH server connected to the auth server.
func (process *TeleportProcess) initSSH() error {
	process.RegisterWithAuthServer(
		process.Config.Token, teleport.RoleNode, SSHIdentityEvent, nil)
	eventsC := make(chan Event)
	process.WaitForEvent(SSHIdentityEvent, eventsC, make(chan struct{}))

	var s *regular.Server

	log := logrus.WithFields(logrus.Fields{
		trace.Component: teleport.ComponentNode,
	})

	process.RegisterFunc("ssh.node", func() error {
		event := <-eventsC
		log.Infof("Received event %q.", event.Name)
		conn, ok := (event.Payload).(*Connector)
		if !ok {
			return trace.BadParameter("unsupported connector type: %T", event.Payload)
		}

		cfg := process.Config

		limiter, err := limiter.NewLimiter(cfg.SSH.Limiter)
		if err != nil {
			return trace.Wrap(err)
		}

		authClient, err := process.newLocalCache(conn.Client, []string{"node"})
		if err != nil {
			return trace.Wrap(err)
		}

		// make sure the namespace exists
		namespace := services.ProcessNamespace(cfg.SSH.Namespace)
		_, err = authClient.GetNamespace(namespace)
		if err != nil {
			if trace.IsNotFound(err) {
				return trace.NotFound(
					"namespace %v is not found, ask your system administrator to create this namespace so you can register nodes there.", namespace)
			}
			return trace.Wrap(err)
		}

		listener, err := process.importOrCreateListener(teleport.ComponentNode, cfg.SSH.Addr.Addr)
		if err != nil {
			return trace.Wrap(err)
		}
		// clean up unused descriptors passed for proxy, but not used by it
		warnOnErr(process.closeImportedDescriptors(teleport.ComponentNode))

		s, err = regular.New(cfg.SSH.Addr,
			cfg.Hostname,
			[]ssh.Signer{conn.Identity.KeySigner},
			authClient,
			cfg.DataDir,
			cfg.AdvertiseIP,
			cfg.Proxy.PublicAddr,
			regular.SetLimiter(limiter),
			regular.SetShell(cfg.SSH.Shell),
			regular.SetAuditLog(conn.Client),
			regular.SetSessionServer(conn.Client),
			regular.SetLabels(cfg.SSH.Labels, cfg.SSH.CmdLabels),
			regular.SetNamespace(namespace),
			regular.SetPermitUserEnvironment(cfg.SSH.PermitUserEnvironment),
			regular.SetCiphers(cfg.Ciphers),
			regular.SetKEXAlgorithms(cfg.KEXAlgorithms),
			regular.SetMACAlgorithms(cfg.MACAlgorithms),
		)
		if err != nil {
			return trace.Wrap(err)
		}

		// init uploader service for recording SSH node, if proxy is not
		// enabled on this node, because proxy stars uploader service as well
		if !cfg.Proxy.Enabled {
			if err := process.initUploaderService(authClient, conn.Client); err != nil {
				return trace.Wrap(err)
			}
		}

		log.Infof("Service is starting on %v %v.", cfg.SSH.Addr.Addr, process.Config.CachePolicy)
		go s.Serve(listener)

		// broadcast that the node has started
		process.BroadcastEvent(Event{Name: NodeSSHReady, Payload: nil})

		// block and wait while the node is running
		s.Wait()
		log.Infof("Exited.")
		return nil
	})
	// execute this when process is asked to exit:
	process.onExit("ssh.shutdown", func(payload interface{}) {
		if payload == nil {
			log.Infof("Shutting down immediately.")
			if s != nil {
				warnOnErr(s.Close())
			}
		} else {
			log.Infof("Shutting down gracefully.")
			if s != nil {
				warnOnErr(s.Shutdown(payloadContext(payload)))
			}
		}
		log.Infof("Exited.")
	})

	return nil
}

// RegisterWithAuthServer uses one time provisioning token obtained earlier
// from the server to get a pair of SSH keys signed by Auth server host
// certificate authority
func (process *TeleportProcess) RegisterWithAuthServer(token string, role teleport.Role, eventName string, additionalPrincipals []string) {
	cfg := process.Config
	identityID := auth.IdentityID{Role: role, HostUUID: cfg.HostUUID, NodeName: cfg.Hostname}

	// this means the server has not been initialized yet, we are starting
	// the registering client that attempts to connect to the auth server
	// and provision the keys
	var authClient *auth.Client
	process.RegisterFunc(fmt.Sprintf("register.%v", strings.ToLower(role.String())), func() error {
		retryTime := defaults.ServerHeartbeatTTL / 3
		for {
			connector, err := process.connectToAuthService(role, additionalPrincipals)
			if err == nil {
				process.BroadcastEvent(Event{Name: eventName, Payload: connector})
				authClient = connector.Client
				return nil
			}
			if trace.IsConnectionProblem(err) {
				log.Infof("%v failed attempt connecting to auth server: %v", role, err)
				time.Sleep(retryTime)
				continue
			}
			if !trace.IsNotFound(err) {
				return trace.Wrap(err)
			}
			//  we haven't connected yet, so we expect the token to exist
			if process.getLocalAuth() != nil {
				// Auth service is on the same host, no need to go though the invitation
				// procedure
				log.Debugf("This server has local Auth server started, using it to add role to the cluster.")
				err = auth.LocalRegister(cfg.DataDir, identityID, process.getLocalAuth(), additionalPrincipals)
			} else {
				// Auth server is remote, so we need a provisioning token
				if token == "" {
					return trace.BadParameter("%v must join a cluster and needs a provisioning token", role)
				}
				log.Infof("Joining the cluster with a token %v.", token)
				err = auth.Register(cfg.DataDir, token, identityID, cfg.AuthServers, additionalPrincipals)
			}
			if err != nil {
				log.Errorf("Failed to join the cluster: %v.", err)
				time.Sleep(retryTime)
			} else {
				log.Infof("%v has successfully registered with the cluster.", role)
				continue
			}
		}
	})

	process.onExit("auth.client", func(interface{}) {
		if authClient != nil {
			authClient.Close()
		}
	})
}

func (process *TeleportProcess) initUploaderService(accessPoint auth.AccessPoint, auditLog events.IAuditLog) error {
	log := logrus.WithFields(logrus.Fields{
		trace.Component: teleport.ComponentAuditLog,
	})
	clusterConfig, err := accessPoint.GetClusterConfig()
	if err != nil {
		return trace.Wrap(err)
	}
	auditConfig := clusterConfig.GetAuditConfig()
	uploadHandler, err := initUploadHandler(auditConfig)
	if err != nil {
		if trace.IsNotFound(err) {
			log.Infof("No session uploader specified, going to ship sessions to auth server.")
			return nil
		}
		if !trace.IsNotFound(err) {
			return trace.Wrap(err)
		}
	}

	// create folder for uploads
	uid, gid, err := adminCreds()
	if err != nil {
		return trace.Wrap(err)
	}

	// prepare dirs for uploader
	path := []string{process.Config.DataDir, teleport.LogsDir, "upload", events.SessionLogsDir, defaults.Namespace}
	for i := 1; i < len(path); i++ {
		dir := filepath.Join(path[:i+1]...)
		log.Infof("Creating directory %v.", dir)
		err := os.Mkdir(dir, 0755)
		err = trace.ConvertSystemError(err)
		if err != nil {
			if !trace.IsAlreadyExists(err) {
				return trace.Wrap(err)
			}
		}
		if uid != nil && gid != nil {
			log.Infof("Setting directory %v owner to %v:%v.", dir, *uid, *gid)
			err := os.Chown(dir, *uid, *gid)
			if err != nil {
				return trace.ConvertSystemError(err)
			}
		}
	}

	log.Infof("Uploader events: %v", process.Config.UploadEventsC)

	uploader, err := events.NewUploader(events.UploaderConfig{
		DataDir:   filepath.Join(process.Config.DataDir, teleport.LogsDir),
		Namespace: defaults.Namespace,
		ServerID:  "upload",
		Handler:   uploadHandler,
		AuditLog:  auditLog,
		EventsC:   process.Config.UploadEventsC,
	})
	if err != nil {
		return trace.Wrap(err)
	}
	process.RegisterFunc("uploader.service", func() error {
		err := uploader.Serve()
		if err != nil {
			log.Errorf("Uploader server exited with error: %v.", err)
		}
		return nil
	})

	process.onExit("uploader.shutdown", func(payload interface{}) {
		log.Infof("Shutting down.")
		warnOnErr(uploader.Stop())
		log.Infof("Exited.")
	})
	return nil
}

// initDiagnosticService starts diagnostic service currently serving healthz
// and prometheus endpoints
func (process *TeleportProcess) initDiagnosticService() error {
	mux := http.NewServeMux()
	mux.Handle("/metrics", prometheus.Handler())

	if process.Config.Debug {
		log.Infof("Adding diagnostic debugging handlers. To connect with profiler, use `go tool pprof %v`.", process.Config.DiagnosticAddr.Addr)

		mux.HandleFunc("/debug/pprof/", pprof.Index)
		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
	}

	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
		roundtrip.ReplyJSON(w, http.StatusOK, map[string]interface{}{"status": "ok"})
	})
	listener, err := process.importOrCreateListener(teleport.ComponentDiagnostic, process.Config.DiagnosticAddr.Addr)
	if err != nil {
		return trace.Wrap(err)
	}
	warnOnErr(process.closeImportedDescriptors(teleport.ComponentDiagnostic))

	server := &http.Server{
		Handler: mux,
	}

	log := logrus.WithFields(logrus.Fields{
		trace.Component: teleport.ComponentDiagnostic,
	})

	log.Infof("Starting diagnostic service on %v.", process.Config.DiagnosticAddr.Addr)

	process.RegisterFunc("diagnostic.service", func() error {
		err := server.Serve(listener)
		if err != nil && err != http.ErrServerClosed {
			log.Warningf("Diagnostic server exited with error: %v.", err)
		}
		return nil
	})

	process.onExit("diagnostic.shutdown", func(payload interface{}) {
		if payload == nil {
			log.Infof("Shutting down immediately.")
			warnOnErr(server.Close())
		} else {
			log.Infof("Shutting down gracefully.")
			ctx := payloadContext(payload)
			warnOnErr(server.Shutdown(ctx))
		}
		log.Infof("Exited.")
	})
	return nil
}

// initProxy gets called if teleport runs with 'proxy' role enabled.
// this means it will do two things:
//    1. serve a web UI
//    2. proxy SSH connections to nodes running with 'node' role
//    3. take care of reverse tunnels
func (process *TeleportProcess) initProxy() error {
	// if no TLS key was provided for the web UI, generate a self signed cert
	if process.Config.Proxy.TLSKey == "" && !process.Config.Proxy.DisableTLS && !process.Config.Proxy.DisableWebService {
		err := initSelfSignedHTTPSCert(process.Config)
		if err != nil {
			return trace.Wrap(err)
		}
	}

	var additionalPrincipals []string
	if process.Config.Proxy.PublicAddr.Addr != "" {
		host, err := utils.Host(process.Config.Proxy.PublicAddr.Addr)
		if err != nil {
			return trace.Wrap(err)
		}
		additionalPrincipals = []string{host}
	}

	process.RegisterWithAuthServer(process.Config.Token, teleport.RoleProxy, ProxyIdentityEvent, additionalPrincipals)
	process.RegisterFunc("proxy.init", func() error {
		eventsC := make(chan Event)
		process.WaitForEvent(ProxyIdentityEvent, eventsC, make(chan struct{}))

		event := <-eventsC
		log.Debugf("Received event %q.", event.Name)
		conn, ok := (event.Payload).(*Connector)
		if !ok {
			return trace.BadParameter("unsupported connector type: %T", event.Payload)
		}

		err := process.initProxyEndpoint(conn)
		if err != nil {
			return trace.Wrap(err)
		}

		return nil
	})
	return nil
}

type proxyListeners struct {
	mux           *multiplexer.Mux
	web           net.Listener
	reverseTunnel net.Listener
}

func (l *proxyListeners) Close() {
	if l.mux != nil {
		l.mux.Close()
	}
	if l.web != nil {
		l.web.Close()
	}
	if l.reverseTunnel != nil {
		l.reverseTunnel.Close()
	}
}

// setupProxyListeners sets up web proxy listeners based on the configuration
func (process *TeleportProcess) setupProxyListeners() (*proxyListeners, error) {
	cfg := process.Config
	log.Debugf("Setup Proxy: Web Proxy Address: %v, Reverse Tunnel Proxy Address: %v", cfg.Proxy.WebAddr.Addr, cfg.Proxy.ReverseTunnelListenAddr.Addr)
	var err error
	var listeners proxyListeners
	switch {
	case cfg.Proxy.DisableWebService && cfg.Proxy.DisableReverseTunnel:
		log.Debugf("Setup Proxy: Reverse tunnel proxy and web proxy are disabled.")
		return &listeners, nil
	case cfg.Proxy.ReverseTunnelListenAddr.Equals(cfg.Proxy.WebAddr):
		log.Debugf("Setup Proxy: Reverse tunnel proxy and web proxy listen on the same port, multiplexing is on.")
		listener, err := process.importOrCreateListener(teleport.Component(teleport.ComponentProxy, "tunnel", "web"), cfg.Proxy.WebAddr.Addr)
		if err != nil {
			return nil, trace.Wrap(err)
		}
		listeners.mux, err = multiplexer.New(multiplexer.Config{
			EnableProxyProtocol: cfg.Proxy.EnableProxyProtocol,
			Listener:            listener,
			DisableTLS:          cfg.Proxy.DisableWebService,
			DisableSSH:          cfg.Proxy.DisableReverseTunnel,
		})
		if err != nil {
			listener.Close()
			return nil, trace.Wrap(err)
		}
		listeners.web = listeners.mux.TLS()
		listeners.reverseTunnel = listeners.mux.SSH()
		go listeners.mux.Serve()
		return &listeners, nil
	case cfg.Proxy.EnableProxyProtocol && !cfg.Proxy.DisableWebService:
		log.Debugf("Setup Proxy: Proxy protocol is enabled for web service, multiplexing is on.")
		listener, err := process.importOrCreateListener(teleport.Component(teleport.ComponentProxy, "web"), cfg.Proxy.WebAddr.Addr)
		if err != nil {
			return nil, trace.Wrap(err)
		}
		listeners.mux, err = multiplexer.New(multiplexer.Config{
			EnableProxyProtocol: cfg.Proxy.EnableProxyProtocol,
			Listener:            listener,
			DisableTLS:          false,
			DisableSSH:          true,
		})
		if err != nil {
			listener.Close()
			return nil, trace.Wrap(err)
		}
		listeners.web = listeners.mux.TLS()
		listeners.reverseTunnel, err = process.importOrCreateListener(teleport.Component(teleport.ComponentProxy, "tunnel"), cfg.Proxy.ReverseTunnelListenAddr.Addr)
		if err != nil {
			listener.Close()
			listeners.Close()
			return nil, trace.Wrap(err)
		}
		go listeners.mux.Serve()
		return &listeners, nil
	default:
		log.Debugf("Proxy reverse tunnel are listening on the separate ports")
		if !cfg.Proxy.DisableReverseTunnel {
			listeners.reverseTunnel, err = process.importOrCreateListener(teleport.Component(teleport.ComponentProxy, "tunnel"), cfg.Proxy.ReverseTunnelListenAddr.Addr)
			if err != nil {
				listeners.Close()
				return nil, trace.Wrap(err)
			}
		}
		if !cfg.Proxy.DisableWebService {
			listeners.web, err = process.importOrCreateListener(teleport.Component(teleport.ComponentProxy, "web"), cfg.Proxy.WebAddr.Addr)
			if err != nil {
				listeners.Close()
				return nil, trace.Wrap(err)
			}
		}
		return &listeners, nil
	}
}

func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error {
	// clean up unused descriptors passed for proxy, but not used by it
	defer process.closeImportedDescriptors(teleport.ComponentProxy)
	var err error
	cfg := process.Config

	proxyLimiter, err := limiter.NewLimiter(cfg.Proxy.Limiter)
	if err != nil {
		return trace.Wrap(err)
	}

	reverseTunnelLimiter, err := limiter.NewLimiter(cfg.Proxy.Limiter)
	if err != nil {
		return trace.Wrap(err)
	}

	// make a caching auth client for the auth server:
	accessPoint, err := process.newLocalCache(conn.Client, []string{"proxy"})
	if err != nil {
		return trace.Wrap(err)
	}

	tlsConfig, err := conn.Identity.TLSConfig()
	if err != nil {
		return trace.Wrap(err)
	}

	listeners, err := process.setupProxyListeners()
	if err != nil {
		return trace.Wrap(err)
	}

	// Register reverse tunnel agents pool
	agentPool, err := reversetunnel.NewAgentPool(reversetunnel.AgentPoolConfig{
		HostUUID:    conn.Identity.ID.HostUUID,
		Client:      conn.Client,
		AccessPoint: accessPoint,
		HostSigners: []ssh.Signer{conn.Identity.KeySigner},
		Cluster:     conn.Identity.Cert.Extensions[utils.CertExtensionAuthority],
	})
	if err != nil {
		return trace.Wrap(err)
	}

	log := logrus.WithFields(logrus.Fields{
		trace.Component: teleport.ComponentReverseTunnelServer,
	})

	// register SSH reverse tunnel server that accepts connections
	// from remote teleport nodes
	var tsrv reversetunnel.Server
	if !process.Config.Proxy.DisableReverseTunnel {
		tsrv, err = reversetunnel.NewServer(
			reversetunnel.Config{
				ID:                    process.Config.HostUUID,
				ClusterName:           conn.Identity.Cert.Extensions[utils.CertExtensionAuthority],
				ClientTLS:             tlsConfig,
				Listener:              listeners.reverseTunnel,
				HostSigners:           []ssh.Signer{conn.Identity.KeySigner},
				LocalAuthClient:       conn.Client,
				LocalAccessPoint:      accessPoint,
				NewCachingAccessPoint: process.newLocalCache,
				Limiter:               reverseTunnelLimiter,
				DirectClusters: []reversetunnel.DirectCluster{
					{
						Name:   conn.Identity.Cert.Extensions[utils.CertExtensionAuthority],
						Client: conn.Client,
					},
				},
				KeyGen:        cfg.Keygen,
				Ciphers:       cfg.Ciphers,
				KEXAlgorithms: cfg.KEXAlgorithms,
				MACAlgorithms: cfg.MACAlgorithms,
				DataDir:       process.Config.DataDir,
			})
		if err != nil {
			return trace.Wrap(err)
		}
		process.RegisterFunc("proxy.reveresetunnel.server", func() error {
			log.Infof("Starting on %v using %v", cfg.Proxy.ReverseTunnelListenAddr.Addr, process.Config.CachePolicy)
			if err := tsrv.Start(); err != nil {
				log.Error(err)
				return trace.Wrap(err)
			}

			// notify parties that we've started reverse tunnel server
			process.BroadcastEvent(Event{Name: ProxyReverseTunnelReady, Payload: tsrv})
			tsrv.Wait()
			return nil
		})
	}

	// Register web proxy server
	var webServer *http.Server
	if !process.Config.Proxy.DisableWebService {
		webHandler, err := web.NewHandler(
			web.Config{
				Proxy:        tsrv,
				AuthServers:  cfg.AuthServers[0],
				DomainName:   cfg.Hostname,
				ProxyClient:  conn.Client,
				DisableUI:    process.Config.Proxy.DisableWebInterface,
				ProxySSHAddr: cfg.Proxy.SSHAddr,
				ProxyWebAddr: cfg.Proxy.WebAddr,
			})
		if err != nil {
			return trace.Wrap(err)
		}
		proxyLimiter.WrapHandle(webHandler)
		if !process.Config.Proxy.DisableTLS {
			log.Infof("Using TLS cert %v, key %v", cfg.Proxy.TLSCert, cfg.Proxy.TLSKey)
			tlsConfig, err := utils.CreateTLSConfiguration(cfg.Proxy.TLSCert, cfg.Proxy.TLSKey)
			if err != nil {
				return trace.Wrap(err)
			}
			listeners.web = tls.NewListener(listeners.web, tlsConfig)
		}
		webServer = &http.Server{
			Handler: proxyLimiter,
		}
		process.RegisterFunc("proxy.web", func() error {
			log.Infof("Web proxy service is starting on %v.", cfg.Proxy.WebAddr.Addr)
			defer webHandler.Close()
			process.BroadcastEvent(Event{Name: ProxyWebServerReady, Payload: webHandler})
			if err := webServer.Serve(listeners.web); err != nil && err != http.ErrServerClosed {
				log.Warningf("Error while serving web requests: %v", err)
			}
			log.Infof("Exited.")
			return nil
		})
	} else {
		log.Infof("Web UI is disabled.")
	}

	// Register SSH proxy server - SSH jumphost proxy server
	listener, err := process.importOrCreateListener(teleport.Component(teleport.ComponentProxy, "ssh"), cfg.Proxy.SSHAddr.Addr)
	if err != nil {
		return trace.Wrap(err)
	}
	sshProxy, err := regular.New(cfg.Proxy.SSHAddr,
		cfg.Hostname,
		[]ssh.Signer{conn.Identity.KeySigner},
		accessPoint,
		cfg.DataDir,
		nil,
		cfg.Proxy.PublicAddr,
		regular.SetLimiter(proxyLimiter),
		regular.SetProxyMode(tsrv),
		regular.SetSessionServer(conn.Client),
		regular.SetAuditLog(conn.Client),
		regular.SetCiphers(cfg.Ciphers),
		regular.SetKEXAlgorithms(cfg.KEXAlgorithms),
		regular.SetMACAlgorithms(cfg.MACAlgorithms),
		regular.SetNamespace(defaults.Namespace),
	)
	if err != nil {
		return trace.Wrap(err)
	}

	process.RegisterFunc("proxy.ssh", func() error {
		log.Infof("SSH proxy service is starting on %v", cfg.Proxy.SSHAddr.Addr)
		go sshProxy.Serve(listener)
		// broadcast that the proxy ssh server has started
		process.BroadcastEvent(Event{Name: ProxySSHReady, Payload: nil})
		return nil
	})

	process.RegisterFunc("proxy.reversetunnel.agent", func() error {
		log := logrus.WithFields(logrus.Fields{
			trace.Component: teleport.ComponentReverseTunnelAgent,
		})
		log.Infof("Starting reverse tunnel agent pool.")
		if err := agentPool.Start(); err != nil {
			log.Errorf("Failed to start: %v.", err)
			return trace.Wrap(err)
		}
		agentPool.Wait()
		return nil
	})

	// execute this when process is asked to exit:
	process.onExit("proxy.shutdown", func(payload interface{}) {
		agentPool.Stop()
		defer listeners.Close()
		if payload == nil {
			log.Infof("Shutting down immediately.")
			if tsrv != nil {
				warnOnErr(tsrv.Close())
			}
			if webServer != nil {
				warnOnErr(webServer.Close())
			}
			warnOnErr(sshProxy.Close())
		} else {
			log.Infof("Shutting down gracefully.")
			ctx := payloadContext(payload)
			if tsrv != nil {
				warnOnErr(tsrv.Shutdown(ctx))
			}
			warnOnErr(sshProxy.Shutdown(ctx))
			if webServer != nil {
				warnOnErr(webServer.Shutdown(ctx))
			}
		}
		log.Infof("Exited.")
	})
	if err := process.initUploaderService(accessPoint, conn.Client); err != nil {
		return trace.Wrap(err)
	}
	return nil
}

func warnOnErr(err error) {
	if err != nil {
		log.Errorf("Error while performing operation: %v", err)
	}
}

// initAuthStorage initializes the storage backend for the auth service.
func (process *TeleportProcess) initAuthStorage() (bk backend.Backend, err error) {
	bc := &process.Config.Auth.StorageConfig

	switch bc.Type {
	// legacy bolt backend:
	case boltbk.GetName():
		bk, err = boltbk.New(bc.Params)
	// filesystem backend:
	case dir.GetName():
		bk, err = dir.New(bc.Params)
	// DynamoDB backend:
	case dynamo.GetName():
		bk, err = dynamo.New(bc.Params)
	// etcd backend:
	case etcdbk.GetName():
		bk, err = etcdbk.New(bc.Params)
	default:
		err = trace.BadParameter("unsupported secrets storage type: %q", bc.Type)
	}
	if err != nil {
		return nil, trace.Wrap(err)
	}
	return bk, nil
}

// StartShutdown launches non-blocking graceful shutdown process that signals
// completion, returns context that will be closed once the shutdown is done
func (process *TeleportProcess) StartShutdown(ctx context.Context) context.Context {
	process.BroadcastEvent(Event{Name: TeleportExitEvent, Payload: ctx})
	localCtx, cancel := context.WithCancel(ctx)
	go func() {
		defer cancel()
		process.Supervisor.Wait()
		log.Debugf("All supervisor functions are completed.")
		localAuth := process.getLocalAuth()
		if localAuth != nil {
			if err := process.localAuth.Close(); err != nil {
				log.Warningf("Failed closing auth server: %v", trace.DebugReport(err))
			}
		}
	}()
	return localCtx
}

// Shutdown launches graceful shutdown process and waits
// for it to complete
func (process *TeleportProcess) Shutdown(ctx context.Context) {
	localCtx := process.StartShutdown(ctx)
	// wait until parent context closes
	select {
	case <-localCtx.Done():
		log.Debugf("Process completed.")
	}
}

// Close broadcasts close signals and exits immediately
func (process *TeleportProcess) Close() error {
	process.BroadcastEvent(Event{Name: TeleportExitEvent})

	process.Config.Keygen.Close()

	localAuth := process.getLocalAuth()
	if localAuth != nil {
		return trace.Wrap(process.localAuth.Close())
	}

	return nil
}

func validateConfig(cfg *Config) error {
	if !cfg.Auth.Enabled && !cfg.SSH.Enabled && !cfg.Proxy.Enabled {
		return trace.BadParameter(
			"config: supply at least one of Auth, SSH or Proxy roles")
	}

	if cfg.DataDir == "" {
		return trace.BadParameter("config: please supply data directory")
	}

	if cfg.Console == nil {
		cfg.Console = ioutil.Discard
	}

	if (cfg.Proxy.TLSKey == "" && cfg.Proxy.TLSCert != "") || (cfg.Proxy.TLSKey != "" && cfg.Proxy.TLSCert == "") {
		return trace.BadParameter("please supply both TLS key and certificate")
	}

	if len(cfg.AuthServers) == 0 {
		return trace.BadParameter("auth_servers is empty")
	}
	for i := range cfg.Auth.Authorities {
		if err := cfg.Auth.Authorities[i].Check(); err != nil {
			return trace.Wrap(err)
		}
	}
	for _, tun := range cfg.ReverseTunnels {
		if err := tun.Check(); err != nil {
			return trace.Wrap(err)
		}
	}

	cfg.SSH.Namespace = services.ProcessNamespace(cfg.SSH.Namespace)

	return nil
}

// initSelfSignedHTTPSCert generates and self-signs a TLS key+cert pair for https connection
// to the proxy server.
func initSelfSignedHTTPSCert(cfg *Config) (err error) {
	log.Warningf("No TLS Keys provided, using self signed certificate.")

	keyPath := filepath.Join(cfg.DataDir, defaults.SelfSignedKeyPath)
	certPath := filepath.Join(cfg.DataDir, defaults.SelfSignedCertPath)

	cfg.Proxy.TLSKey = keyPath
	cfg.Proxy.TLSCert = certPath

	// return the existing pair if they have already been generated:
	_, err = tls.LoadX509KeyPair(certPath, keyPath)
	if err == nil {
		return nil
	}
	if !os.IsNotExist(err) {
		return trace.Wrap(err, "unrecognized error reading certs")
	}
	log.Warningf("Generating self signed key and cert to %v %v.", keyPath, certPath)

	creds, err := utils.GenerateSelfSignedCert([]string{cfg.Hostname, "localhost"})
	if err != nil {
		return trace.Wrap(err)
	}

	if err := ioutil.WriteFile(keyPath, creds.PrivateKey, 0600); err != nil {
		return trace.Wrap(err, "error writing key PEM")
	}
	if err := ioutil.WriteFile(certPath, creds.Cert, 0600); err != nil {
		return trace.Wrap(err, "error writing key PEM")
	}
	return nil
}