Add vectored I/O support for Windows.

Open vfds in "overlapped" mode, and teach pg_pread()/pg_pwrite() to
wait if necessary.  Add pg_direct_preadv()/pg_direct_pwritev() as
synonyms for pg_preadv()/pg_pwritev() when using a direct I/O file
handle, but on Windows redirect them to ReadFileScatter() and
WriteFileGather().  Teach fd.c to use those alternative names if a File
was opened with PG_O_DIRECT, since it remembers that for vfds.

This is enough for io_method=sync and io_method=worker to benefit from
true vectored I/O for buffer pool I/O when debug_io_direct=data is used
on this platform.  Later patches for native AIO need this too.

Windows doesn't have vectored APIs for buffered I/O, and still falls
back to the loop-based emulation in that case.

Author: macdice

src/backend/storage/file/fd.c

@@ -1621,6 +1621,17 @@ PathNameOpenFilePerm(const char *fileName, int fileFlags, mode_t fileMode)
 	 */
 	fileFlags |= O_CLOEXEC;
 
+#ifdef WIN32
+
+	/*
+	 * Enable overlapped for all vfds on Windows.  This and PG_O_DIRECT are
+	 * required for vectored I/O.  The pg_p{read,readv,write,writev}()
+	 * emulation functions wait for completion if required, so they remain
+	 * fully synchronous.
+	 */
+	fileFlags |= O_OVERLAPPED;
+#endif
+
 	vfdP->fd = BasicOpenFilePerm(fileName, fileFlags, fileMode);
 
 	if (vfdP->fd < 0)
@@ -2183,7 +2194,10 @@ FileReadV(File file, const struct iovec *iov, int iovcnt, off_t offset,
 
 retry:
 	pgstat_report_wait_start(wait_event_info);
-	returnCode = pg_preadv(vfdP->fd, iov, iovcnt, offset);
+	if (vfdP->fileFlags & PG_O_DIRECT)
+		returnCode = pg_direct_preadv(vfdP->fd, iov, iovcnt, offset);
+	else
+		returnCode = pg_preadv(vfdP->fd, iov, iovcnt, offset);
 	pgstat_report_wait_end();
 
 	if (returnCode < 0)
@@ -2293,7 +2307,10 @@ FileWriteV(File file, const struct iovec *iov, int iovcnt, off_t offset,
 
 retry:
 	pgstat_report_wait_start(wait_event_info);
-	returnCode = pg_pwritev(vfdP->fd, iov, iovcnt, offset);
+	if (vfdP->fileFlags & PG_O_DIRECT)
+		returnCode = pg_direct_pwritev(vfdP->fd, iov, iovcnt, offset);
+	else
+		returnCode = pg_pwritev(vfdP->fd, iov, iovcnt, offset);
 	pgstat_report_wait_end();
 
 	if (returnCode >= 0)

src/include/port.h

@@ -338,7 +338,8 @@ extern bool rmtree(const char *path, bool rmtopdir);
  * open() and fopen() replacements to allow deletion of open files and
  * passing of other special options.
  */
-#define		O_DIRECT	0x80000000
+#define		O_OVERLAPPED	0x40000000
+#define		O_DIRECT		0x80000000
 extern HANDLE pgwin32_open_handle(const char *, int, bool);
 extern int	pgwin32_open(const char *, int,...);
 extern FILE *pgwin32_fopen(const char *, const char *);

src/include/port/pg_iovec.h

@@ -40,6 +40,18 @@ struct iovec
  */
 #define PG_IOV_MAX Min(IOV_MAX, 128)
 
+/* Expose whether vectored I/O is emulated on this platform. */
+#if defined(WIN32)
+/* Windows only has it for O_DIRECT | O_OVERLAPPED. */
+#define pg_iov_emulated(direct) (!(direct))
+#elif HAVE_DECL_PREADV && HAVE_DECL_PWRITEV
+/* In practice this covers all modern Unixen. */
+#define pg_iov_emulated(direct) false
+#else
+/* A few historical late adopters, not likely in the field. */
+#define pg_iov_emulated(direct) true
+#endif
+
 /*
  * Like preadv(), but with a prefix to remind us of a side-effect: on Windows
  * this changes the current file position.
@@ -118,4 +130,30 @@ pg_pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset)
 #endif
 }
 
+/*
+ * Variants for callers that opened with O_DIRECT and (Windows only)
+ * O_OVERLAPPED.  This reaches real vectored I/O on Windows, but fails without
+ * those flags.
+ */
+
+static inline ssize_t
+pg_direct_preadv(int fd, const struct iovec *iov, int iovcnt, off_t offset)
+{
+#ifdef WIN32
+	if (iovcnt > 1)
+		return pg_win32_direct_preadv(fd, iov, iovcnt, offset);
+#endif
+	return pg_preadv(fd, iov, iovcnt, offset);
+}
+
+static inline ssize_t
+pg_direct_pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset)
+{
+#ifdef WIN32
+	if (iovcnt > 1)
+		return pg_win32_direct_pwritev(fd, iov, iovcnt, offset);
+#endif
+	return pg_pwritev(fd, iov, iovcnt, offset);
+}
+
 #endif							/* PG_IOVEC_H */

src/include/port/win32_port.h

@@ -585,8 +585,27 @@ typedef unsigned short mode_t;
 
 /* in port/win32pread.c */
 extern ssize_t pg_pread(int fd, void *buf, size_t nbyte, off_t offset);
+extern ssize_t pg_win32_direct_preadv(int fd, struct iovec *iovec, int iovcnt, off_t offset);
 
 /* in port/win32pwrite.c */
 extern ssize_t pg_pwrite(int fd, const void *buf, size_t nbyte, off_t offset);
+extern ssize_t pg_win32_direct_pwritev(int fd, struct iovec *iovec, int iovcnt, off_t offset);
+
+/*
+ * This should in theory be GetSystemInfo()'s dwPageSize, but it's
+ * PG_IO_ALIGN_SIZE on all known modern systems.  If it's ever wrong, direct
+ * I/O operations will fail here and elsewhere, and you'll have to turn that
+ * mode off, but that applies to every OS.
+ */
+#define PG_WIN32_FILE_SEGMENT_SIZE PG_IO_ALIGN_SIZE
+
+/* Max segments expected for block-oriented I/O. */
+#define PG_WIN32_FILE_SEGMENTS_MAX (PG_IOV_MAX * (BLCKSZ / PG_IO_ALIGN_SIZE))
+
+/* in port/win32common.c */
+extern DWORD pg_win32_iovec_to_file_segments(FILE_SEGMENT_ELEMENT * segments,
+											 int maxsegments,
+											 struct iovec *iov,
+											 int iovcnt);
 
 #endif							/* PG_WIN32_PORT_H */

src/port/open.c

@@ -75,6 +75,7 @@ pgwin32_open_handle(const char *fileName, int fileFlags, bool backup_semantics)
 	assert((fileFlags & ((O_RDONLY | O_WRONLY | O_RDWR) | O_APPEND |
 						 (O_RANDOM | O_SEQUENTIAL | O_TEMPORARY) |
 						 _O_SHORT_LIVED | O_DSYNC | O_DIRECT |
+						 O_OVERLAPPED |
 						 (O_CREAT | O_TRUNC | O_EXCL) | (O_TEXT | O_BINARY))) == fileFlags);
 
 	sa.nLength = sizeof(sa);
@@ -95,6 +96,7 @@ pgwin32_open_handle(const char *fileName, int fileFlags, bool backup_semantics)
 						   ((fileFlags & O_SEQUENTIAL) ? FILE_FLAG_SEQUENTIAL_SCAN : 0) |
 						   ((fileFlags & _O_SHORT_LIVED) ? FILE_ATTRIBUTE_TEMPORARY : 0) |
 						   ((fileFlags & O_TEMPORARY) ? FILE_FLAG_DELETE_ON_CLOSE : 0) |
+						   ((fileFlags & O_OVERLAPPED) ? FILE_FLAG_OVERLAPPED : 0) |
 						   ((fileFlags & O_DIRECT) ? FILE_FLAG_NO_BUFFERING : 0) |
 						   ((fileFlags & O_DSYNC) ? FILE_FLAG_WRITE_THROUGH : 0),
 						   NULL)) == INVALID_HANDLE_VALUE)

src/port/win32common.c

@@ -19,6 +19,8 @@
 #include "postgres.h"
 #endif
 
+#include "port/pg_iovec.h"
+
 /*
  * pgwin32_get_file_type
  *
@@ -62,3 +64,39 @@ pgwin32_get_file_type(HANDLE hFile)
 
 	return fileType;
 }
+
+/*
+ * Windows scatter/gather works with lists of raw page addresses, which this
+ * function produces from Unix iovec format.  All iovecs must be aligned to
+ * PG_WIN32_FILE_SEGMENT_SIZE (but they always are in PostgreSQL when using
+ * direct I/O, and vectored I/O is only available on Windows with direct I/O).
+ * Returns zero on badly aligned or input that would exceed maxsegments.
+ */
+DWORD
+pg_win32_iovec_to_file_segments(FILE_SEGMENT_ELEMENT * segments,
+								int maxsegments,
+								struct iovec *iov,
+								int iovcnt)
+{
+	DWORD		nsegments = 0;
+
+	for (int i = 0; i < iovcnt; ++i)
+	{
+		char	   *base = iov[i].iov_base;
+		size_t		len = iov[i].iov_len;
+
+		if (nsegments > maxsegments ||
+			((intptr_t) base) % PG_WIN32_FILE_SEGMENT_SIZE != 0 ||
+			len % PG_WIN32_FILE_SEGMENT_SIZE != 0)
+			return 0;
+
+		while (len > 0)
+		{
+			segments[nsegments++].Buffer = base;
+			base += PG_WIN32_FILE_SEGMENT_SIZE;
+			len -= PG_WIN32_FILE_SEGMENT_SIZE;
+		}
+	}
+
+	return nsegments * PG_WIN32_FILE_SEGMENT_SIZE;
+}

src/port/win32pread.c

@@ -13,9 +13,22 @@
 
 
 #include "c.h"
+#include "port/pg_iovec.h"
 
 #include <windows.h>
 
+#ifndef FRONTEND
+static HANDLE
+completion_event(void)
+{
+	static HANDLE h = NULL;
+
+	if (!h && !(h = CreateEvent(NULL, true, false, NULL)))
+		errno = ENOMEM;
+	return h;
+}
+#endif
+
 ssize_t
 pg_pread(int fd, void *buf, size_t size, off_t offset)
 {
@@ -33,10 +46,81 @@ pg_pread(int fd, void *buf, size_t size, off_t offset)
 	/* Avoid overflowing DWORD. */
 	size = Min(size, 1024 * 1024 * 1024);
 
-	/* Note that this changes the file position, despite not using it. */
+	/*
+	 * Note that this changes the file position despite not using it unless
+	 * opened with O_OVERLAPPED.
+	 */
 	overlapped.Offset = offset;
+
+#ifndef FRONTEND
+	/* If not synchronously handled from cache, we may need to wait. */
+	if (unlikely(!(overlapped.hEvent = completion_event())))
+		return -1;
+#endif
+
 	if (!ReadFile(handle, buf, size, &result, &overlapped))
 	{
+#ifndef FRONTEND
+		if (GetLastError() == ERROR_IO_PENDING &&
+			GetOverlappedResult(overlapped.hEvent,
+								&overlapped,
+								&result,
+								TRUE))
+			return result;
+#endif
+
+		if (GetLastError() == ERROR_HANDLE_EOF)
+			return 0;
+
+		_dosmaperr(GetLastError());
+		return -1;
+	}
+
+	return result;
+}
+
+#ifndef FRONTEND
+/*
+ * Special emulation of preadv() that works with O_DIRECT | O_OVERLAPPED.
+ * See pg_iovec.h for general emulation.
+ */
+ssize_t
+pg_win32_direct_preadv(int fd, struct iovec *iov, int iovcnt, off_t offset)
+{
+	FILE_SEGMENT_ELEMENT segments[PG_WIN32_FILE_SEGMENTS_MAX];
+	OVERLAPPED	overlapped = {0};
+	HANDLE		handle;
+	DWORD		size;
+	DWORD		result;
+
+	handle = (HANDLE) _get_osfhandle(fd);
+	if (handle == INVALID_HANDLE_VALUE)
+	{
+		errno = EBADF;
+		return -1;
+	}
+
+	size = pg_win32_iovec_to_file_segments(segments, lengthof(segments),
+										   iov, iovcnt);
+	if (size == 0)
+	{
+		errno = EINVAL;
+		return -1;
+	}
+
+	/* If not synchronously handled from cache, we may need to wait. */
+	if (unlikely(!(overlapped.hEvent = completion_event())))
+		return -1;
+
+	if (!ReadFileScatter(handle, segments, size, &result, &overlapped))
+	{
+		if (GetLastError() == ERROR_IO_PENDING &&
+			GetOverlappedResult(overlapped.hEvent,
+								&overlapped,
+								&result,
+								TRUE))
+			return result;
+
 		if (GetLastError() == ERROR_HANDLE_EOF)
 			return 0;
 
@@ -46,3 +130,4 @@ pg_pread(int fd, void *buf, size_t size, off_t offset)
 
 	return result;
 }
+#endif