FUSE ioctl returns error even if returning 0 from ioctl handler

[felfert]

Bug Report

I am currently implementing ioctl functionality in a proprietary fuse driver.
My ioctl_handler gets called and works as expected, however despite returning 0
from the ioctl handler, the calling DeviceIoControl(...) always returns false with
GetLastError() returning 1 (Incorrect function)

How to Reproduce

• I prepared a modified passthrough-fuse, adding a similar ROT13 ioctl like
  in memfs. This is available here:
  passthrough-fuse.c
• I also wrote a small client program for manual testing:

#include <windows.h>
#include <string>
#include <iostream>

#define FSP_FUSE_DEVICE_TYPE            (0x8000 | 'W' | 'F' * 0x100) /* DeviceIoControl -> ioctl */
#define FSP_FUSE_CTLCODE_FROM_IOCTL(cmd) \
    (FSP_FUSE_DEVICE_TYPE << 16) | (((cmd) & 0x0fff) << 2)

using namespace std;

static string emsg(DWORD ecode) {
    LPVOID lpMsgBuf = 0;
    if (0 == ::FormatMessageA(
            FORMAT_MESSAGE_ALLOCATE_BUFFER|FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS,
            NULL, ecode, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL)) {
        return string("Unknown error");
    }
    string ret(static_cast<LPCSTR>(lpMsgBuf));
    ::LocalFree(lpMsgBuf);
    while (ret.size() && (ret[ret.size()-1] == '\n' || ret[ret.size()-1] == '\r')) {
        ret.erase(ret.size()-1);
    }
    if (ret.size() && ret[ret.size()-1] == '.') {
        ret.erase(ret.size()-1);
    }
    return ret;
}

int main(int argc, char **argv) {
    if (argc < 3) {
        cerr << "Usage: bioctl <fuse|memfs> <path>" << endl;
        return 1;
    }
	if (strcmp("fuse", argv[1]) && strcmp("memfs", argv[1])) {
		cerr << "FS-type must be fuse or memfs" << endl;
		return 1;
	}
    HANDLE h = ::CreateFileA(argv[2], GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING,
            FILE_FLAG_BACKUP_SEMANTICS, 0);
    if (INVALID_HANDLE_VALUE == h) {
        cerr << "CreateFileA(" << argv[2] << ") returned an invalid handle" << endl;
        return 1;
    }
	CHAR Buffer[26];
    DWORD BytesTransferred = 0;
	DWORD ccode;
	if (strcmp("fuse", argv[1])) {
		ccode = CTL_CODE(0x8000 + 'M', 'R', METHOD_BUFFERED, FILE_ANY_ACCESS);
	} else {
		ccode = FSP_FUSE_CTLCODE_FROM_IOCTL('f');
	}
    cout << "calling DeviceIoControl with code=0x" << hex << ccode << endl;
    if (!::DeviceIoControl(h, ccode, (LPVOID)"ABCDEFghijklmNOPQRStuvwxyz", 26,
                Buffer, sizeof(Buffer), &BytesTransferred, 0)) {
        DWORD err = GetLastError();
        cerr << "DeviceIoControl failed: " << emsg(err) << " (0x" << err << ")" << endl;
        return 1;
    }
    CloseHandle(h);
    if (BytesTransferred != 26) {
        cerr << "BytesTransferred != 26 (" << BytesTransferred << ")" << endl;
    }
    if (memcmp("NOPQRStuvwxyzABCDEFghijklm", Buffer, 26)) {
        cerr << "Buffer did not contain \"NOPQRStuvwxyzABCDEFghijklm\"" << endl;
    }
    return 0;
}

• To demonstrate the bug, first run the modified passthrough-fuse.exe, then invoke
  the above program like this

.\bioctl.exe fuse <SomePathBelowTheFuseMountedDrive> 

Behaviors

The above program will print

calling DeviceIoControl with code=0xc6570198
DeviceIoControl failed: Incorrect function (0x1)

Environment

• OS version and build:
  Output of diag.bat attached:
  diag.bat.log

• WinFsp version and build:
  1.4.19016