Use the right functions to deal with WINAPI HANDLEs + refactoring

Author: Vincent Blondeau

platforms/win32/plugins/FilePlugin/sqWin32FilePrims.c

@@ -286,8 +286,15 @@ sqConnectToFile(SQFile *sqFile, void *file, sqInt writeFlag)
 	return 1;
 }
 
-
-
+void
+sqFileStdioHandlesIntoFile_WithHandle_IsWritable(SQFile file, HANDLE handle, int isWritable) {
+	file.sessionID = thisSession;
+	file.file = handle;
+	file.writable = isWritable;
+	file.lastOp = 0; /* unused on win32 */
+	file.isStdioStream = isFileHandleATTY(handle);
+	AddHandleToTable(win32Files, handle);
+}
 
 /*
  * Fill-in files with handles for stdin, stdout and seterr as available and
@@ -297,32 +304,15 @@ sqConnectToFile(SQFile *sqFile, void *file, sqInt writeFlag)
 sqInt
 sqFileStdioHandlesInto(SQFile files[3])
 {
-	DWORD mode;
-
-	files[0].sessionID = thisSession;
-	files[0].file = GetStdHandle(STD_INPUT_HANDLE);
-	files[0].writable = false;
-	files[0].lastOp = 0; /* unused on win32 */
-	files[0].isStdioStream = isFileHandleATTY(STD_INPUT_HANDLE);
-	AddHandleToTable(win32Files, files[0].file);
-
-	files[1].sessionID = thisSession;
-	files[1].file = GetStdHandle(STD_OUTPUT_HANDLE);
-	files[1].writable = true;
-	files[1].lastOp = 0; /* unused on win32 */
-	files[1].isStdioStream = isFileHandleATTY(STD_OUTPUT_HANDLE);
-	AddHandleToTable(win32Files, files[1].file);
-
-	files[2].sessionID = thisSession;
-	files[2].file = GetStdHandle(STD_ERROR_HANDLE);
-	files[2].writable = true;
-	files[2].lastOp = 0; /* unused on win32 */
-	files[2].isStdioStream = isFileHandleATTY(STD_ERROR_HANDLE);
-	AddHandleToTable(win32Files, files[2].file);
+	sqFileStdioHandlesIntoFile_WithHandle_IsWritable(files[0], GetStdHandle(STD_INPUT_HANDLE), false);
+	sqFileStdioHandlesIntoFile_WithHandle_IsWritable(files[1], GetStdHandle(STD_OUTPUT_HANDLE), true);
+	sqFileStdioHandlesIntoFile_WithHandle_IsWritable(files[2], GetStdHandle(STD_ERROR_HANDLE), true);
 
 	return 7;
 }
 
+
+
 /* 
  * Allow to test if the standard input/output files are from a console or not
  * 1 if stdio is redirected to a console pipe, else 0 (and in this case, a file should be created)

platforms/win32/vm/sqWin32Main.c

@@ -876,13 +876,24 @@ getVersionInfo(int verbose)
  * Inspired of: https://fossies.org/linux/misc/vim-8.0.tar.bz2/vim80/src/iscygpty.c?m=t
  */
 sqInt  isFileHandleATTY(HANDLE fdHandle) {
-	//In case of Windows Shell case
-	int res = _isatty(_fileno(fdHandle));
-	if (res != 0)
-		return res < 0;
-	if (errno == EBADF)	
+	if (fdHandle == INVALID_HANDLE_VALUE) {
 		return 0;
-	//In case of Unix emulator, we parse the name of the pipe
+	}
+	
+	/* In case of Windows Shell case */
+	DWORD fileType = GetFileType(fdHandle);
+	if (fileType == FILE_TYPE_CHAR)
+		/* The specified file is a character file, typically an LPT device or a console. */
+		/* https://msdn.microsoft.com/en-us/library/windows/desktop/aa364960(v=vs.85).aspx */
+		return 1;
+
+	/* In case of Unix emulator, we need to parse the name of the pipe */
+	
+	/* Cygwin/msys's pty is a pipe. */
+	if (fileType != FILE_TYPE_PIPE) {
+		return 0;
+	}
+	
 	int size = sizeof(FILE_NAME_INFO) + sizeof(WCHAR) * MAX_PATH;
 	FILE_NAME_INFO *nameinfo;
 	WCHAR *p = NULL;
@@ -900,13 +911,7 @@ sqInt  isFileHandleATTY(HANDLE fdHandle) {
 		if (!pGetFileInformationByHandleEx)
 			return 0;
 	}
-	if (fdHandle == INVALID_HANDLE_VALUE) {
-		return 0;
-	}
-	/* Cygwin/msys's pty is a pipe. */
-	if (GetFileType(fdHandle) != FILE_TYPE_PIPE) {
-		return 0;
-	}
+
 	nameinfo = malloc(size);
 	if (nameinfo == NULL) {
 		return 0;
@@ -932,7 +937,8 @@ sqInt  isFileHandleATTY(HANDLE fdHandle) {
 * 1 if one of the stdio is redirected to a console pipe, else 0 (and in this case, a file should be created)
 */
 sqInt  isOneStdioDescriptorATTY() {
-	return isFileHandleATTY(STD_INPUT_HANDLE) || isFileHandleATTY(STD_OUTPUT_HANDLE) || isFileHandleATTY(STD_ERROR_HANDLE);
+	return isFileHandleATTY(GetStdHandle(STD_INPUT_HANDLE)) || 
+		isFileHandleATTY(GetStdHandle(STD_OUTPUT_HANDLE)) || isFileHandleATTY(GetStdHandle(STD_ERROR_HANDLE));
 }
 
 static void