syscall: add SysProcAttr.PseudoConsole on Windows

This allows the user to pass a ConPty handle to run a process on a
ConPty session.

See https://learn.microsoft.com/en-us/windows/console/creating-a-pseudoconsole-session
Fixes #62708

src/syscall/exec_windows.go

@@ -251,6 +251,7 @@ type SysProcAttr struct {
 	NoInheritHandles           bool                // if set, no handles are inherited by the new process, not even the standard handles, contained in ProcAttr.Files, nor the ones contained in AdditionalInheritedHandles
 	AdditionalInheritedHandles []Handle            // a list of additional handles, already marked as inheritable, that will be inherited by the new process
 	ParentProcess              Handle              // if non-zero, the new process regards the process given by this handle as its parent process, and AdditionalInheritedHandles, if set, should exist in this parent process
+	PseudoConsole              Handle              // if non-zero, the new process will be attached to the console represented by this handle, any AdditionalInheritedHandles will be ignored, this implies NoInheritHandles
 }
 
 var zeroProcAttr ProcAttr
@@ -371,9 +372,13 @@ func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid int, handle
 			return 0, 0, err
 		}
 	}
-	si.StdInput = fd[0]
-	si.StdOutput = fd[1]
-	si.StdErr = fd[2]
+
+	// If a PseudoConsole is specified, then there is nothing we need to do with the handles since the process will inherit the other end of the PseudoConsole.
+	if sys.PseudoConsole == 0 {
+		si.StdInput = fd[0]
+		si.StdOutput = fd[1]
+		si.StdErr = fd[2]
+	}
 
 	fd = append(fd, sys.AdditionalInheritedHandles...)
 
@@ -396,7 +401,7 @@ func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid int, handle
 	}
 	fd = fd[:j]
 
-	willInheritHandles := len(fd) > 0 && !sys.NoInheritHandles
+	willInheritHandles := len(fd) > 0 && !sys.NoInheritHandles && sys.PseudoConsole == 0
 
 	// Do not accidentally inherit more than these handles.
 	if willInheritHandles {
@@ -406,6 +411,13 @@ func StartProcess(argv0 string, argv []string, attr *ProcAttr) (pid int, handle
 		}
 	}
 
+	if sys.PseudoConsole != 0 {
+		err = updateProcThreadAttribute(si.ProcThreadAttributeList, 0, _PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE, unsafe.Pointer(sys.PseudoConsole), unsafe.Sizeof(sys.PseudoConsole), nil, nil)
+		if err != nil {
+			return 0, 0, err
+		}
+	}
+
 	envBlock, err := createEnvBlock(attr.Env)
 	if err != nil {
 		return 0, 0, err

src/syscall/exec_windows_test.go

@@ -5,13 +5,17 @@
 package syscall_test
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"testing"
 	"time"
+	"unsafe"
 )
 
 func TestEscapeArg(t *testing.T) {
@@ -113,3 +117,97 @@ func TestChangingProcessParent(t *testing.T) {
 		t.Fatalf("child output: want %q, got %q", want, got)
 	}
 }
+
+func TestPseudoConsoleProcess(t *testing.T) {
+	pty, err := newConPty()
+	if err != nil {
+		t.Errorf("create pty failed: %v", err)
+	}
+
+	defer pty.Close()
+	cmd := exec.Command("cmd")
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		PseudoConsole: syscall.Handle(pty.handle),
+	}
+
+	if err := cmd.Start(); err != nil {
+		t.Errorf("start cmd failed: %v", err)
+	}
+
+	var outBuf bytes.Buffer
+	go func() { pty.inPipe.Write([]byte("exit\r\n")) }()
+	go io.Copy(&outBuf, pty.outPipe)
+
+	_ = cmd.Wait()
+
+	if got, want := outBuf.String(), "Microsoft Windows"; !strings.Contains(got, want) {
+		t.Errorf("cmd output: want %q, got %q", want, got)
+	}
+}
+
+var (
+	kernel32 = syscall.MustLoadDLL("kernel32.dll")
+
+	procCreatePseudoConsole = kernel32.MustFindProc("CreatePseudoConsole")
+	procClosePseudoConsole  = kernel32.MustFindProc("ClosePseudoConsole")
+)
+
+type conPty struct {
+	handle  syscall.Handle
+	inPipe  *os.File
+	outPipe *os.File
+}
+
+func (c *conPty) Close() error {
+	closePseudoConsole(c.handle)
+	if err := c.inPipe.Close(); err != nil {
+		return err
+	}
+	return c.outPipe.Close()
+}
+
+// See https://learn.microsoft.com/en-us/windows/console/creating-a-pseudoconsole-session
+func newConPty() (*conPty, error) {
+	inputRead, inputWrite, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+
+	outputRead, outputWrite, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+
+	var handle syscall.Handle
+	coord := uint32(25<<16) | 80 // 80x25 screen buffer
+	err = createPseudoConsole(coord, syscall.Handle(inputRead.Fd()), syscall.Handle(outputWrite.Fd()), 0, &handle)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := outputWrite.Close(); err != nil {
+		return nil, err
+	}
+	if err := inputRead.Close(); err != nil {
+		return nil, err
+	}
+
+	return &conPty{
+		handle:  handle,
+		inPipe:  inputWrite,
+		outPipe: outputRead,
+	}, nil
+}
+
+func createPseudoConsole(size uint32, in syscall.Handle, out syscall.Handle, flags uint32, pconsole *syscall.Handle) (hr error) {
+	r0, _, _ := syscall.Syscall6(procCreatePseudoConsole.Addr(), 5, uintptr(size), uintptr(in), uintptr(out), uintptr(flags), uintptr(unsafe.Pointer(pconsole)), 0)
+	if r0 != 0 {
+		hr = syscall.Errno(r0)
+	}
+	return
+}
+
+func closePseudoConsole(console syscall.Handle) {
+	syscall.Syscall(procClosePseudoConsole.Addr(), 1, uintptr(console), 0, 0)
+	return
+}

src/syscall/types_windows.go

@@ -497,6 +497,7 @@ type _PROC_THREAD_ATTRIBUTE_LIST struct {
 const (
 	_PROC_THREAD_ATTRIBUTE_PARENT_PROCESS = 0x00020000
 	_PROC_THREAD_ATTRIBUTE_HANDLE_LIST    = 0x00020002
+	_PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE  = 0x00020016
 )
 
 type _STARTUPINFOEXW struct {