os: implement os.fd_is_pending/1, os.Process.pipe_read/1, os.Process.…

…is_pending/1 (#19787)

.github/workflows/cross_ci.yml

@@ -76,7 +76,7 @@ jobs:
       - name: v_win.c can be compiled and run with -os windows
         run: |
           ./v -cc msvc -os windows -o /tmp/v_win.c cmd/v
-          x86_64-w64-mingw32-gcc /tmp/v_win.c -std=c99 -w -municode -o v_from_vc.exe
+          x86_64-w64-mingw32-gcc /tmp/v_win.c -std=c99 -w -municode -o v_from_vc.exe -lws2_32
           ls -lart v_from_vc.exe
           wine64 ./v_from_vc.exe version
 

.github/workflows/windows_ci.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Test new v.c
         run: |
           .\v.exe -o v.c cmd/v
-          gcc -Werror -municode -w v.c
+          gcc -Werror -municode -w v.c -lws2_32
       - name: Install dependencies
         run: |
           .\v.exe setup-freetype
@@ -119,7 +119,7 @@ jobs:
       - name: Test new v.c
         run: |
           .\v.exe -o v.c cmd/v
-          .\thirdparty\tcc\tcc.exe -Werror -w -ladvapi32 -bt10 v.c
+          .\thirdparty\tcc\tcc.exe -Werror -w -ladvapi32 -lws2_32 -bt10 v.c
       - name: Install dependencies
         run: |
           .\v.exe setup-freetype
@@ -163,7 +163,7 @@ jobs:
       #     .\v.exe wipe-cache
       #     .\make.bat -tcc32
       # - name: Test new v.c
-      #   run: .\v.exe -o v.c cmd/v && .\thirdparty\tcc\tcc.exe -Werror -g -w -ladvapi32 -bt10 v.c
+      #   run: .\v.exe -o v.c cmd/v && .\thirdparty\tcc\tcc.exe -Werror -g -w -ladvapi32 -lws2_32 -bt10 v.c
       # - name: v doctor
       #   run: ./v doctor
       #

Dockerfile.cross

@@ -4,7 +4,7 @@ LABEL maintainer="Delyan Angelov <delian66@gmail.com>"
 COPY . .
 RUN make
 RUN ./v -os windows -o v.c cmd/v
-RUN x86_64-w64-mingw32-gcc v.c -std=c99 -w -municode -o v.exe
+RUN x86_64-w64-mingw32-gcc v.c -std=c99 -w -municode -o v.exe -lws2_32
 RUN file v.exe
 
 CMD [ "bash" ]

GNUmakefile

@@ -98,7 +98,7 @@ endif
 
 all: latest_vc latest_tcc latest_legacy
 ifdef WIN32
-	$(CC) $(CFLAGS) -std=c99 -municode -w -o v1.exe $(VC)/$(VCFILE) $(LDFLAGS)
+	$(CC) $(CFLAGS) -std=c99 -municode -w -o v1.exe $(VC)/$(VCFILE) $(LDFLAGS) -lws2_32
 	v1.exe -no-parallel -o v2.exe $(VFLAGS) cmd/v
 	v2.exe -o $(VEXE) $(VFLAGS) cmd/v
 	del v1.exe

cmd/tools/modules/vgit/vgit.v

@@ -198,6 +198,10 @@ pub fn (mut vgit_context VGitContext) compile_oldv_if_needed() {
 			// after 53ffee1 2020-05-18, gcc builds on windows do need `-municode`
 			c_flags += '-municode'
 		}
+		// after 2023-11-07, windows builds need linking to ws2_32:
+		if vgit_context.commit_v__ts >= 1699341818 && !vgit_context.cc.contains('msvc') {
+			c_flags += '-lws2_32'
+		}
 		command_for_building_v_from_c_source = '${vgit_context.cc} ${c_flags} -o cv.exe  "${vc_source_file_location}" ${c_ldflags}'
 		command_for_selfbuilding = '.\\cv.exe -o ${vgit_context.vexename} {SOURCE}'
 	} else {

examples/process/write_and_read_from_a_bash_child_process.v

@@ -0,0 +1,97 @@
+module main
+
+// This example shows how to communicate with a child process (`bash` in this case), by sending
+// commands to its stdin pipe, and reading responses from its stdout and stderr pipes.
+// Note, you can use `if p.is_pending(.stdout) {` and `if p.is_pending(.stderr) {`, to check if
+// there is available data in the pipes, without having to block in your main loop, if the data
+// is missing or just not available yet.
+import os
+import time
+
+const tmp_folder = os.join_path(os.temp_dir(), 'process_folder')
+
+const max_txt_files = 20
+
+fn exec(cmd string) (string, int, string) {
+	mut out := []string{}
+	mut er := []string{}
+	mut rc := 0
+
+	mut p := os.new_process('/bin/bash')
+	p.set_redirect_stdio()
+	p.run()
+
+	p.stdin_write('echo "START         " && sleep 0.1 && ${cmd};\n')
+	p.stdin_write('ECODE=\$?;\n')
+	p.stdin_write('sleep 0.1;\n')
+	p.stdin_write('exit \$ECODE;\n')
+
+	// Note, that you can also ensure that `bash` will exit, when the command finishes,
+	// by closing its stdin pipe. In the above example, that is not needed however, since
+	// the last `exit` command, will make it quit as well.
+	// os.fd_close(p.stdio_fd[0])
+
+	for p.is_alive() {
+		if data := p.pipe_read(.stderr) {
+			eprintln('p.pipe_read .stderr, len: ${data.len:4} | data: `${data#[0..10]}`...')
+			er << data
+		}
+		if data := p.pipe_read(.stdout) {
+			eprintln('p.pipe_read .stdout, len: ${data.len:4} | data: `${data#[0..10]}`...')
+			out << data
+		}
+		// avoid a busy loop, by sleeping a bit between each iteration
+		time.sleep(2 * time.millisecond)
+	}
+
+	// the process finished, slurp all the remaining data in the pipes:
+	out << p.stdout_slurp()
+	er << p.stderr_slurp()
+	p.close()
+	p.wait()
+
+	if p.code > 0 {
+		eprintln('----------------------------------------------------------')
+		eprintln('COMMAND: ${cmd}')
+		eprintln('STDOUT:\n${out}')
+		eprintln('STDERR:\n${er}')
+		eprintln('----------------------------------------------------------')
+		rc = 1
+	}
+
+	return out.join(''), rc, er.join('')
+}
+
+fn main() {
+	mut out := ''
+	mut er := ''
+	mut ecode := 0
+
+	// prepare some files in a temporary folder
+	defer {
+		os.rmdir_all(tmp_folder) or {}
+	}
+	os.mkdir_all(tmp_folder) or {}
+	for i in 0 .. max_txt_files {
+		os.write_file(os.join_path(tmp_folder, '${i}.txt'), '${i}\n${i}\n')!
+	}
+
+	out, ecode, er = exec("find ${os.quoted_path(tmp_folder)} ; sleep 0.1; find ${os.quoted_path(tmp_folder)} ; echo '******'")
+	assert out.ends_with('******\n')
+	assert er == ''
+
+	out, ecode, er = exec('echo to stdout')
+	assert out.contains('to stdout')
+	assert er == ''
+
+	out, ecode, er = exec('echo to stderr 1>&2')
+	assert out.starts_with('START')
+	assert er.contains('to stderr')
+
+	out, ecode, er = exec('ls /sssss')
+	assert out.starts_with('START')
+	assert er != ''
+	assert ecode > 0 // THIS STILL GIVES AN ERROR !
+
+	println('test ok stderr & stdout is indeed redirected, ecode: ${ecode}')
+}

make.bat

@@ -134,7 +134,7 @@ REM By default, use tcc, since we have it prebuilt:
 :tcc_strap
 :tcc32_strap
 echo  ^> Attempting to build "%V_BOOTSTRAP%" (from v_win.c) with "!tcc_exe!"
-"!tcc_exe!" -Bthirdparty/tcc -bt10 -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32
+"!tcc_exe!" -Bthirdparty/tcc -bt10 -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32 -lws2_32
 if %ERRORLEVEL% NEQ 0 goto :compile_error
 echo  ^> Compiling "%V_EXE%" with "%V_BOOTSTRAP%"
 "%V_BOOTSTRAP%" -keepc -g -showcc -cc "!tcc_exe!" -cflags -Bthirdparty/tcc -o "%V_UPDATED%" cmd/v
@@ -151,7 +151,7 @@ if %ERRORLEVEL% NEQ 0 (
 )
 
 echo  ^> Attempting to build "%V_BOOTSTRAP%" (from v_win.c) with Clang
-clang -std=c99 -municode -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32
+clang -std=c99 -municode -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32 -lws2_32
 if %ERRORLEVEL% NEQ 0 (
 	echo In most cases, compile errors happen because the version of Clang installed is too old
 	clang --version
@@ -173,7 +173,7 @@ if %ERRORLEVEL% NEQ 0 (
 )
 
 echo  ^> Attempting to build "%V_BOOTSTRAP%" (from v_win.c) with GCC
-gcc -std=c99 -municode -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32
+gcc -std=c99 -municode -g -w -o "%V_BOOTSTRAP%" ./vc/v_win.c -ladvapi32 -lws2_32
 if %ERRORLEVEL% NEQ 0 (
 	echo In most cases, compile errors happen because the version of GCC installed is too old
 	gcc --version
@@ -214,7 +214,7 @@ if exist "%InstallDir%/Common7/Tools/vsdevcmd.bat" (
 set ObjFile=.v.c.obj
 
 echo  ^> Attempting to build "%V_BOOTSTRAP%" (from v_win.c) with MSVC
-cl.exe /volatile:ms /Fo%ObjFile% /W0 /MD /D_VBOOTSTRAP "vc/v_win.c" user32.lib kernel32.lib advapi32.lib shell32.lib /link /nologo /out:"%V_BOOTSTRAP%" /incremental:no
+cl.exe /volatile:ms /Fo%ObjFile% /W0 /MD /D_VBOOTSTRAP "vc/v_win.c" user32.lib kernel32.lib advapi32.lib shell32.lib ws2_32.lib /link /nologo /out:"%V_BOOTSTRAP%" /incremental:no
 if %ERRORLEVEL% NEQ 0 (
 	echo In some cases, compile errors happen because of the MSVC compiler version
 	cl.exe

vlib/os/fd.c.v

@@ -1,15 +1,27 @@
 module os
 
-// file descriptor based operations:
+// low level operations with file descriptors/handles
 
-// close filedescriptor
+$if !windows {
+	#include <sys/select.h>
+}
+
+$if windows {
+	#include <winsock2.h>
+}
+
+#flag windows -lws2_32
+
+// fd_close closes the file descriptor. It returns 0 on success.
 pub fn fd_close(fd int) int {
 	if fd == -1 {
 		return 0
 	}
 	return C.close(fd)
 }
 
+// fd_write writes the given string to the file descriptor.
+// It blocks until all the data in the string is written.
 pub fn fd_write(fd int, s string) {
 	if fd == -1 {
 		return
@@ -26,7 +38,7 @@ pub fn fd_write(fd int, s string) {
 	}
 }
 
-// read from filedescriptor, block until data
+// fd_slurp reads all the remaining data from the file descriptor.
 pub fn fd_slurp(fd int) []string {
 	mut res := []string{}
 	if fd == -1 {
@@ -42,8 +54,7 @@ pub fn fd_slurp(fd int) []string {
 	return res
 }
 
-// read from filedescriptor, don't block
-// return [bytestring,nrbytes]
+// fd_read reads data from the file descriptor. It returns the read data, and how many bytes were read.
 pub fn fd_read(fd int, maxbytes int) (string, int) {
 	if fd == -1 {
 		return '', 0
@@ -59,3 +70,37 @@ pub fn fd_read(fd int, maxbytes int) (string, int) {
 		return tos(buf, nbytes), nbytes
 	}
 }
+
+[typedef]
+pub struct C.fd_set {}
+
+pub struct C.timeval {
+	tv_sec  u64
+	tv_usec u64
+}
+
+fn C.@select(ndfs int, readfds &C.fd_set, writefds &C.fd_set, exceptfds &C.fd_set, timeout &C.timeval) int
+
+// These are C macros, but from the V's point of view, can be treated as C functions:
+fn C.FD_ZERO(fdset &C.fd_set)
+fn C.FD_SET(fd int, fdset &C.fd_set)
+fn C.FD_ISSET(fd int, fdset &C.fd_set) int
+
+// fd_is_pending returns true, when there is pending data, waiting to be read from file descriptor `fd`.
+// If the file descriptor is closed, or if reading from it, will block (there is no data), fd_is_pending returns false.
+pub fn fd_is_pending(fd int) bool {
+	read_set := C.fd_set{}
+	C.FD_ZERO(&read_set)
+	C.FD_SET(fd, &read_set)
+	mut ts := C.timeval{
+		tv_sec: 0
+		tv_usec: 0
+	}
+	res := C.@select(fd + 1, &read_set, C.NULL, C.NULL, &ts)
+	if res > 0 {
+		if C.FD_ISSET(fd, &read_set) != 0 {
+			return true
+		}
+	}
+	return false
+}

vlib/os/file_test.v

@@ -93,9 +93,9 @@ fn test_read_bytes_into_newline_binary() {
 	bw[9] = 0xff
 	bw[12] = 10 // newline
 
-	n0_bytes := bw[0..10]
-	n1_bytes := bw[10..13]
-	n2_bytes := bw[13..]
+	n0_bytes := unsafe { bw[0..10] }
+	n1_bytes := unsafe { bw[10..13] }
+	n2_bytes := unsafe { bw[13..] }
 
 	mut f := os.open_file(tfile, 'w')!
 	f.write(bw)!