Win32: fix broken pipe detection

As of "Win32: Thread-safe windows console output", git-log no longer
terminates when the pager process dies. This is due to disabling buffering
for the replaced stdout / stderr streams. Git-log will periodically fflush
stdout (see write_or_die.c/mayble_flush_or_die()), but with no buffering,
this is a NOP that always succeeds (so we never detect the EPIPE error).

Exchange the original console handles with our console thread pipe handles
by accessing the internal MSVCRT data structures directly (which are
exposed via __pioinfo for some reason).

Implement this with minimal assumptions about the actual data structure to
make it work with different (hopefully even future) MSVCRT versions.

While messing with internal data structures is ugly, this patch solves the
problem at the source instead of adding more workarounds. We no longer need
the special winansi_isatty override, and the limitations documented in
"Win32: Thread-safe windows console output" are gone (i.e. fdopen(1/2)
returns unbuffered streams now, and isatty() for duped console file
descriptors works as expected).

Signed-off-by: Karsten Blees <blees@dcon.de>

compat/mingw.h

@@ -331,9 +331,7 @@ int mingw_raise(int sig);
  */
 
 void winansi_init(void);
-int winansi_isatty(int fd);
 HANDLE winansi_get_osfhandle(int fd);
-#define isatty winansi_isatty
 
 /*
  * git specific compatibility

compat/winansi.c

@@ -7,11 +7,6 @@
 #include <wingdi.h>
 #include <winreg.h>
 
-/*
- Functions to be wrapped:
-*/
-#undef isatty
-
 /*
  ANSI codes used by git: m, K
 
@@ -103,6 +98,7 @@ static int is_console(int fd)
 
 	/* initialize attributes */
 	if (!initialized) {
+		console = hcon;
 		attr = plain_attr = sbi.wAttributes;
 		negative = 0;
 		initialized = 1;
@@ -463,29 +459,80 @@ static HANDLE duplicate_handle(HANDLE hnd)
 	return hresult;
 }
 
-static HANDLE redirect_console(FILE *stream, HANDLE *phcon, int new_fd)
-{
-	/* get original console handle */
-	int fd = _fileno(stream);
-	HANDLE hcon = (HANDLE) _get_osfhandle(fd);
-	if (hcon == INVALID_HANDLE_VALUE)
-		die_errno("_get_osfhandle(%i) failed", fd);
 
-	/* save a copy to phcon and console (used by the background thread) */
-	console = *phcon = duplicate_handle(hcon);
+/*
+ * Make MSVCRT's internal file descriptor control structure accessible
+ * so that we can tweak OS handles and flags directly (we need MSVCRT
+ * to treat our pipe handle as if it were a console).
+ *
+ * We assume that the ioinfo structure (exposed by MSVCRT.dll via
+ * __pioinfo) starts with the OS handle and the flags. The exact size
+ * varies between MSVCRT versions, so we try different sizes until
+ * toggling the FDEV bit of _pioinfo(1)->osflags is reflected in
+ * isatty(1).
+ */
+typedef struct {
+	HANDLE osfhnd;
+	char osflags;
+} ioinfo;
+
+extern __declspec(dllimport) ioinfo *__pioinfo[];
 
-	/* duplicate new_fd over fd (closes fd and associated handle (hcon)) */
-	if (_dup2(new_fd, fd))
-		die_errno("_dup2(%i, %i) failed", new_fd, fd);
+static size_t sizeof_ioinfo = 0;
 
-	/* no buffering, or stdout / stderr will be out of sync */
-	setbuf(stream, NULL);
-	return (HANDLE) _get_osfhandle(fd);
+#define IOINFO_L2E 5
+#define IOINFO_ARRAY_ELTS (1 << IOINFO_L2E)
+
+#define FDEV  0x40
+
+static inline ioinfo* _pioinfo(int fd)
+{
+	return (ioinfo*)((char*)__pioinfo[fd >> IOINFO_L2E] +
+			(fd & (IOINFO_ARRAY_ELTS - 1)) * sizeof_ioinfo);
+}
+
+static int init_sizeof_ioinfo()
+{
+	int istty, wastty;
+	/* don't init twice */
+	if (sizeof_ioinfo)
+		return sizeof_ioinfo >= 256;
+
+	sizeof_ioinfo = sizeof(ioinfo);
+	wastty = isatty(1);
+	while (sizeof_ioinfo < 256) {
+		/* toggle FDEV flag, check isatty, then toggle back */
+		_pioinfo(1)->osflags ^= FDEV;
+		istty = isatty(1);
+		_pioinfo(1)->osflags ^= FDEV;
+		/* return if we found the correct size */
+		if (istty != wastty)
+			return 0;
+		sizeof_ioinfo += sizeof(void*);
+	}
+	error("Tweaking file descriptors doesn't work with this MSVCRT.dll");
+	return 1;
+}
+
+static HANDLE swap_osfhnd(int fd, HANDLE new_handle)
+{
+	ioinfo *pioinfo;
+	HANDLE old_handle;
+
+	/* init ioinfo size if we haven't done so */
+	if (init_sizeof_ioinfo())
+		return INVALID_HANDLE_VALUE;
+
+	/* get ioinfo pointer and change the handles */
+	pioinfo = _pioinfo(fd);
+	old_handle = pioinfo->osfhnd;
+	pioinfo->osfhnd = new_handle;
+	return old_handle;
 }
 
 void winansi_init(void)
 {
-	int con1, con2, hwrite_fd;
+	int con1, con2;
 	char name[32];
 
 	/* check if either stdout or stderr is a console output screen buffer */
@@ -514,39 +561,18 @@ void winansi_init(void)
 	if (atexit(winansi_exit))
 		die_errno("atexit(winansi_exit) failed");
 
-	/* create a file descriptor for the write end of the pipe */
-	hwrite_fd = _open_osfhandle((long) duplicate_handle(hwrite), _O_BINARY);
-	if (hwrite_fd == -1)
-		die_errno("_open_osfhandle(%li) failed", (long) hwrite);
-
 	/* redirect stdout / stderr to the pipe */
 	if (con1)
-		hwrite1 = redirect_console(stdout, &hconsole1, hwrite_fd);
+		hconsole1 = swap_osfhnd(1, hwrite1 = duplicate_handle(hwrite));
 	if (con2)
-		hwrite2 = redirect_console(stderr, &hconsole2, hwrite_fd);
-
-	/* close pipe file descriptor (also closes the duped hwrite) */
-	close(hwrite_fd);
+		hconsole2 = swap_osfhnd(2, hwrite2 = duplicate_handle(hwrite));
 }
 
 static int is_same_handle(HANDLE hnd, int fd)
 {
 	return hnd != INVALID_HANDLE_VALUE && hnd == (HANDLE) _get_osfhandle(fd);
 }
 
-/*
- * Return true if stdout / stderr is a pipe redirecting to the console.
- */
-int winansi_isatty(int fd)
-{
-	if (fd == 1 && is_same_handle(hwrite1, 1))
-		return 1;
-	else if (fd == 2 && is_same_handle(hwrite2, 2))
-		return 1;
-	else
-		return isatty(fd);
-}
-
 /*
  * Returns the real console handle if stdout / stderr is a pipe redirecting
  * to the console. Allows spawn / exec to pass the console to the next process.