Introduce disconnect client logic.

This commit implements #1935, fixes #2038

Auth server now supports global
defaults for timeout behavior:

```
auth_service:
  client_idle_timeout:  15m
  disconnect_expired_cert: no
```

New role options were introduced:

```
kind: role
version: v3
metadata:
  name: intern
  spec:
    options:
    # these two settings override the global ones:
    client_idle_timeout:  1m
    disconnect_expired_cert: yes
```

constants.go

@@ -299,6 +299,9 @@ const (
 	// CertificateFormatUnspecified is used to check if the format was specified
 	// or not.
 	CertificateFormatUnspecified = ""
+
+	// DurationNever is human friendly shortcut that is interpreted as a Duration of 0
+	DurationNever = "never"
 )
 
 const (

integration/helpers.go

@@ -354,7 +354,7 @@ func SetupUser(process *service.TeleportProcess, username string, roles []servic
 
 		// allow tests to forward agent, still needs to be passed in client
 		roleOptions := role.GetOptions()
-		roleOptions.Set(services.ForwardAgent, true)
+		roleOptions.ForwardAgent = services.NewBool(true)
 		role.SetOptions(roleOptions)
 
 		err = auth.UpsertRole(role, backend.Forever)
@@ -508,7 +508,7 @@ func (i *TeleInstance) CreateEx(trustedSecrets []*InstanceSecrets, tconf *servic
 
 			// allow tests to forward agent, still needs to be passed in client
 			roleOptions := role.GetOptions()
-			roleOptions.Set(services.ForwardAgent, true)
+			roleOptions.ForwardAgent = services.NewBool(true)
 			role.SetOptions(roleOptions)
 
 			err = auth.UpsertRole(role, backend.Forever)

integration/integration_test.go

@@ -491,26 +491,26 @@ func (s *IntSuite) TestInteroperability(c *check.C) {
 		// 0 - echo "1\n2\n" | ssh localhost "cat -"
 		// this command can be used to copy files by piping stdout to stdin over ssh.
 		{
-			"cat -",
-			"1\n2\n",
-			"1\n2\n",
-			false,
+			inCommand:   "cat -",
+			inStdin:     "1\n2\n",
+			outContains: "1\n2\n",
+			outFile:     false,
 		},
 		// 1 - ssh -tt locahost '/bin/sh -c "mkdir -p /tmp && echo a > /tmp/file.txt"'
 		// programs like ansible execute commands like this
 		{
-			fmt.Sprintf(`/bin/sh -c "mkdir -p /tmp && echo a > %v"`, tempfile),
-			"",
-			"a",
-			true,
+			inCommand:   fmt.Sprintf(`/bin/sh -c "mkdir -p /tmp && echo a > %v"`, tempfile),
+			inStdin:     "",
+			outContains: "a",
+			outFile:     true,
 		},
 		// 2 - ssh localhost tty
 		// should print "not a tty"
 		{
-			"tty",
-			"",
-			"not a tty",
-			false,
+			inCommand:   "tty",
+			inStdin:     "",
+			outContains: "not a tty",
+			outFile:     false,
 		},
 	}
 
@@ -521,7 +521,7 @@ func (s *IntSuite) TestInteroperability(c *check.C) {
 
 		// hook up stdin and stdout to a buffer for reading and writing
 		inbuf := bytes.NewReader([]byte(tt.inStdin))
-		outbuf := &bytes.Buffer{}
+		outbuf := utils.NewSyncBuffer()
 		cl.Stdin = inbuf
 		cl.Stdout = outbuf
 		cl.Stderr = outbuf
@@ -688,6 +688,153 @@ func (s *IntSuite) TestShutdown(c *check.C) {
 	}
 }
 
+type disconnectTestCase struct {
+	recordingMode     string
+	options           services.RoleOptions
+	disconnectTimeout time.Duration
+}
+
+// TestDisconnectScenarios tests multiple scenarios with client disconnects
+func (s *IntSuite) TestDisconnectScenarios(c *check.C) {
+
+	testCases := []disconnectTestCase{
+		{
+			recordingMode: services.RecordAtNode,
+			options: services.RoleOptions{
+				ClientIdleTimeout: services.NewDuration(500 * time.Millisecond),
+			},
+			disconnectTimeout: time.Second,
+		},
+		{
+			recordingMode: services.RecordAtProxy,
+			options: services.RoleOptions{
+				ClientIdleTimeout: services.NewDuration(500 * time.Millisecond),
+			},
+			disconnectTimeout: time.Second,
+		},
+		{
+			recordingMode: services.RecordAtNode,
+			options: services.RoleOptions{
+				DisconnectExpiredCert: services.NewBool(true),
+				MaxSessionTTL:         services.NewDuration(2 * time.Second),
+			},
+			disconnectTimeout: 4 * time.Second,
+		},
+		{
+			recordingMode: services.RecordAtProxy,
+			options: services.RoleOptions{
+				DisconnectExpiredCert: services.NewBool(true),
+				MaxSessionTTL:         services.NewDuration(2 * time.Second),
+			},
+			disconnectTimeout: 4 * time.Second,
+		},
+	}
+	for _, tc := range testCases {
+		s.runDisconnectTest(c, tc)
+	}
+}
+
+func (s *IntSuite) runDisconnectTest(c *check.C, tc disconnectTestCase) {
+	t := NewInstance(InstanceConfig{
+		ClusterName: Site,
+		HostID:      HostID,
+		NodeName:    Host,
+		Ports:       s.getPorts(5),
+		Priv:        s.priv,
+		Pub:         s.pub,
+	})
+
+	// devs role gets disconnected after 1 second idle time
+	username := s.me.Username
+	role, err := services.NewRole("devs", services.RoleSpecV3{
+		Options: tc.options,
+		Allow: services.RoleConditions{
+			Logins: []string{username},
+		},
+	})
+	c.Assert(err, check.IsNil)
+	t.AddUserWithRole(username, role)
+
+	clusterConfig, err := services.NewClusterConfig(services.ClusterConfigSpecV3{
+		SessionRecording: services.RecordAtNode,
+	})
+	c.Assert(err, check.IsNil)
+
+	cfg := service.MakeDefaultConfig()
+	cfg.Auth.Enabled = true
+	cfg.Auth.ClusterConfig = clusterConfig
+	cfg.Proxy.DisableWebService = true
+	cfg.Proxy.DisableWebInterface = true
+	cfg.Proxy.Enabled = true
+	cfg.SSH.Enabled = true
+
+	c.Assert(t.CreateEx(nil, cfg), check.IsNil)
+	c.Assert(t.Start(), check.IsNil)
+	defer t.Stop(true)
+
+	// get a reference to site obj:
+	site := t.GetSiteAPI(Site)
+	c.Assert(site, check.NotNil)
+
+	person := NewTerminal(250)
+
+	// commandsC receive commands
+	commandsC := make(chan string, 0)
+
+	// PersonA: SSH into the server, wait one second, then type some commands on stdin:
+	sessionCtx, sessionCancel := context.WithCancel(context.TODO())
+	openSession := func() {
+		defer sessionCancel()
+		cl, err := t.NewClient(ClientConfig{Login: username, Cluster: Site, Host: Host, Port: t.GetPortSSHInt()})
+		c.Assert(err, check.IsNil)
+		cl.Stdout = &person
+		cl.Stdin = &person
+
+		go func() {
+			for command := range commandsC {
+				person.Type(command)
+			}
+		}()
+
+		err = cl.SSH(context.TODO(), []string{}, false)
+		if err != nil && err != io.EOF {
+			c.Fatalf("expected EOF or nil, got %v instead", err)
+		}
+	}
+
+	go openSession()
+
+	retry := func(command, pattern string) {
+		person.Type(command)
+		abortTime := time.Now().Add(10 * time.Second)
+		var matched bool
+		var output string
+		for {
+			output = string(replaceNewlines(person.Output(1000)))
+			matched, _ = regexp.MatchString(pattern, output)
+			if matched {
+				break
+			}
+			time.Sleep(time.Millisecond * 200)
+			if time.Now().After(abortTime) {
+				c.Fatalf("failed to capture output: %v", pattern)
+			}
+		}
+		if !matched {
+			c.Fatalf("output %q does not match pattern %q", output, pattern)
+		}
+	}
+
+	retry("echo start \r\n", ".*start.*")
+	time.Sleep(tc.disconnectTimeout)
+	select {
+	case <-time.After(tc.disconnectTimeout):
+		c.Fatalf("timeout waiting for session to exit")
+	case <-sessionCtx.Done():
+		// session closed
+	}
+}
+
 // TestInvalidLogins validates that you can't login with invalid login or
 // with invalid 'site' parameter
 func (s *IntSuite) TestEnvironmentVariables(c *check.C) {
@@ -1509,7 +1656,7 @@ func (s *IntSuite) TestDiscovery(c *check.C) {
 	// attempt to allow the discovery request to be received and the connection
 	// added to the agent pool.
 	lb.AddBackend(mainProxyAddr)
-	output, err = runCommand(main, []string{"echo", "hello world"}, cfg, 10)
+	output, err = runCommand(main, []string{"echo", "hello world"}, cfg, 20)
 	c.Assert(err, check.IsNil)
 	c.Assert(output, check.Equals, "hello world\n")
 

lib/auth/permissions.go

@@ -387,7 +387,7 @@ func GetCheckerForBuiltinRole(clusterName string, clusterConfig services.Cluster
 			role.String(),
 			services.RoleSpecV3{
 				Options: services.RoleOptions{
-					services.MaxSessionTTL: services.MaxDuration(),
+					MaxSessionTTL: services.MaxDuration(),
 				},
 				Allow: services.RoleConditions{
 					Namespaces: []string{services.Wildcard},

lib/auth/tls_test.go

@@ -1060,7 +1060,7 @@ func (s *TLSSuite) TestGenerateCerts(c *check.C) {
 
 	// now update role to permit agent forwarding
 	roleOptions := userRole.GetOptions()
-	roleOptions.Set(services.ForwardAgent, true)
+	roleOptions.ForwardAgent = services.NewBool(true)
 	userRole.SetOptions(roleOptions)
 	err = s.server.Auth().UpsertRole(userRole, backend.Forever)
 	c.Assert(err, check.IsNil)
@@ -1124,7 +1124,7 @@ func (s *TLSSuite) TestCertificateFormat(c *check.C) {
 
 	for _, tt := range tests {
 		roleOptions := userRole.GetOptions()
-		roleOptions.Set(services.CertificateFormat, tt.inRoleCertificateFormat)
+		roleOptions.CertificateFormat = tt.inRoleCertificateFormat
 		userRole.SetOptions(roleOptions)
 		err := s.server.Auth().UpsertRole(userRole, backend.Forever)
 		c.Assert(err, check.IsNil)

lib/config/configuration.go

@@ -443,9 +443,11 @@ func ApplyFileConfig(fc *FileConfig, cfg *service.Config) error {
 
 	// build cluster config from session recording and host key checking preferences
 	cfg.Auth.ClusterConfig, err = services.NewClusterConfig(services.ClusterConfigSpecV3{
-		SessionRecording:    fc.Auth.SessionRecording,
-		ProxyChecksHostKeys: fc.Auth.ProxyChecksHostKeys,
-		Audit:               *auditConfig,
+		SessionRecording:      fc.Auth.SessionRecording,
+		ProxyChecksHostKeys:   fc.Auth.ProxyChecksHostKeys,
+		Audit:                 *auditConfig,
+		ClientIdleTimeout:     fc.Auth.ClientIdleTimeout,
+		DisconnectExpiredCert: fc.Auth.DisconnectExpiredCert,
 	})
 	if err != nil {
 		return trace.Wrap(err)

lib/config/configuration_test.go

@@ -119,6 +119,61 @@ func (s *ConfigTestSuite) TestSampleConfig(c *check.C) {
 	c.Assert(lib.IsInsecureDevMode(), check.Equals, false)
 }
 
+// TestBooleanParsing tests that boolean options
+// are parsed properly
+func (s *ConfigTestSuite) TestBooleanParsing(c *check.C) {
+	testCases := []struct {
+		s string
+		b bool
+	}{
+		{s: "true", b: true},
+		{s: "'true'", b: true},
+		{s: "yes", b: true},
+		{s: "'yes'", b: true},
+		{s: "'1'", b: true},
+		{s: "1", b: true},
+		{s: "no", b: false},
+		{s: "0", b: false},
+	}
+	for i, tc := range testCases {
+		comment := check.Commentf("test case %v", i)
+		conf, err := ReadFromString(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`
+teleport:
+  advertise_ip: 10.10.10.1
+auth_service:
+  enabled: yes
+  disconnect_expired_cert: %v
+`, tc.s))))
+		c.Assert(err, check.IsNil)
+		c.Assert(conf.Auth.DisconnectExpiredCert.Value(), check.Equals, tc.b, comment)
+	}
+}
+
+// TestDurationParsing tests that duration options
+// are parsed properly
+func (s *ConfigTestSuite) TestDuration(c *check.C) {
+	testCases := []struct {
+		s string
+		d time.Duration
+	}{
+		{s: "1s", d: time.Second},
+		{s: "never", d: 0},
+		{s: "'1m'", d: time.Minute},
+	}
+	for i, tc := range testCases {
+		comment := check.Commentf("test case %v", i)
+		conf, err := ReadFromString(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(`
+teleport:
+  advertise_ip: 10.10.10.1
+auth_service:
+  enabled: yes
+  client_idle_timeout: %v
+`, tc.s))))
+		c.Assert(err, check.IsNil)
+		c.Assert(conf.Auth.ClientIdleTimeout.Value(), check.Equals, tc.d, comment)
+	}
+}
+
 func (s *ConfigTestSuite) TestConfigReading(c *check.C) {
 	// non-existing file:
 	conf, err := ReadFromFile("/heaven/trees/apple.ymL")
@@ -149,6 +204,8 @@ func (s *ConfigTestSuite) TestConfigReading(c *check.C) {
 	c.Assert(conf.Auth.Enabled(), check.Equals, true)
 	c.Assert(conf.Auth.ListenAddress, check.Equals, "tcp://auth")
 	c.Assert(conf.Auth.LicenseFile, check.Equals, "lic.pem")
+	c.Assert(conf.Auth.DisconnectExpiredCert.Value(), check.Equals, true)
+	c.Assert(conf.Auth.ClientIdleTimeout.Value(), check.Equals, 17*time.Second)
 	c.Assert(conf.SSH.Configured(), check.Equals, true)
 	c.Assert(conf.SSH.Enabled(), check.Equals, true)
 	c.Assert(conf.SSH.ListenAddress, check.Equals, "tcp://ssh")
@@ -575,6 +632,8 @@ func makeConfigFixture() string {
 	conf.Auth.EnabledFlag = "Yeah"
 	conf.Auth.ListenAddress = "tcp://auth"
 	conf.Auth.LicenseFile = "lic.pem"
+	conf.Auth.ClientIdleTimeout = services.NewDuration(17 * time.Second)
+	conf.Auth.DisconnectExpiredCert = services.NewBool(true)
 
 	// ssh service:
 	conf.SSH.EnabledFlag = "true"