Access violation exception in msys-2.0.dll in ssh.exe

[Northcode]

I have a rather strange error running ssh.exe with any valid arguments causes it to throw an access violation exception during password input, debugging with visual studio gives the following info:

Exception thrown at 0x000000018012B647 (msys-2.0.dll) in ssh.exe: 0xC0000005: Access violation writing location 0x0000000000000000.

The surrounding disassembly looks like:

000000018012B62C  mov         rax,8  
000000018012B633  xadd        qword ptr [r10-1BB0h],rax  
000000018012B63B  lea         r11,[18012B65Bh]  
000000018012B642  xchg        r11,qword ptr [rsp+8]  
000000018012B647  mov         qword ptr [rax],r11  
000000018012B64A  inc         dword ptr [r10-1BBCh]  
000000018012B651  dec         dword ptr [r10-1BB4h]  
000000018012B658  pop         rax  
000000018012B659  jmp         rax  
000000018012B65B  mov         r10,qword ptr gs:[8]  
000000018012B664  mov         r11d,1  
000000018012B66A  xchg        r11d,dword ptr [r10-1BB4h]  
000000018012B671  mov         dword ptr [r10-1BB8h],r11d  
000000018012B678  test        r11d,r11d  
000000018012B67B  je          000000018012B681  

Strangely enough if I start ssh.exe with "Step into new instance" from visual studio, it works, and does not throw the exception.
Starting it from the msys2 shell in either mintty, conemu or cmd causes it to just crash during password input, breaking the terminal (it gets stuck in password mode, a quick reset fixes that but it will still crash again every time).

Strace of ssh sdf.org:

create_child: ssh sdf.org
--- Process 21908 created
--- Process 21908 loaded C:\Windows\System32\ntdll.dll at 0000000076e10000
--- Process 21908 loaded C:\Windows\System32\kernel32.dll at 0000000076bf0000
--- Process 21908 loaded C:\Windows\System32\KernelBase.dll at 000007fefcb60000
--- Process 21908 loaded C:\msys64\usr\bin\msys-crypto-1.1.dll at 000000058b980000
--- Process 21908 loaded C:\msys64\usr\bin\msys-2.0.dll at 0000000180040000
--- Process 21908 loaded C:\msys64\usr\bin\msys-z.dll at 0000000522fe0000
--- Process 21908 loaded C:\msys64\usr\bin\msys-gssapi-3.dll at 00000004f53c0000
--- Process 21908 loaded C:\msys64\usr\bin\msys-asn1-8.dll at 00000004901d0000
--- Process 21908 loaded C:\msys64\usr\bin\msys-com_err-1.dll at 0000000419420000
--- Process 21908 loaded C:\msys64\usr\bin\msys-roken-18.dll at 00000005fe2a0000
--- Process 21908 loaded C:\msys64\usr\bin\msys-crypt-0.dll at 000000043dbf0000
--- Process 21908 loaded C:\msys64\usr\bin\msys-heimntlm-0.dll at 00000005fe970000
--- Process 21908 loaded C:\msys64\usr\bin\msys-krb5-26.dll at 00000004e6ec0000
--- Process 21908 loaded C:\msys64\usr\bin\msys-heimbase-1.dll at 00000005889a0000
--- Process 21908 loaded C:\msys64\usr\bin\msys-wind-0.dll at 0000000497010000
--- Process 21908 loaded C:\msys64\usr\bin\msys-hx509-5.dll at 000000047f610000
--- Process 21908 loaded C:\msys64\usr\bin\msys-hcrypto-4.dll at 00000004a4450000
--- Process 21908 loaded C:\msys64\usr\bin\msys-gcc_s-seh-1.dll at 00000005e8160000
--- Process 21908 loaded C:\msys64\usr\bin\msys-sqlite3-0.dll at 00000005798a0000
    0       0 [main] ssh (21908) **********************************************
 1149    1149 [main] ssh (21908) Program name: C:\msys64\usr\bin\ssh.exe (windows pid 21908)
  455    1604 [main] ssh (21908) OS version:   Windows NT-6.1
  419    2023 [main] ssh (21908) **********************************************
--- Process 21908 loaded C:\Windows\System32\advapi32.dll at 000007fefe6b0000
--- Process 21908 loaded C:\Windows\System32\msvcrt.dll at 000007fefed70000
--- Process 21908 loaded C:\Windows\System32\sechost.dll at 000007fefeb50000
--- Process 21908 loaded C:\Windows\System32\rpcrt4.dll at 000007fefe450000
--- Process 21908 loaded C:\Windows\System32\cryptbase.dll at 000007fefc5c0000
92885   94908 [main] ssh (21908) sigprocmask: 0 = sigprocmask (0, 0x0, 0x18030DD30)
  526   95434 [main] ssh (21908) open_shared: name shared.5, n 5, shared 0x180030000 (wanted 0x180030000), h 0x88, *m 6
  264   95698 [main] ssh (21908) user_heap_info::init: heap base 0x600000000, heap top 0x600000000, heap size 0x20000000 (536870912)
  520   96218 [main] ssh (21908) open_shared: name S-1-5-21-1796241753-961618004-8547516-351679.1, n 1, shared 0x180020000 (wanted 0x180020000), h 0x8C, *m 6
  496   96714 [main] ssh (21908) user_info::create: opening user shared for 'S-1-5-21-1796241753-961618004-8547516-351679' at 0x180020000
  520   97234 [main] ssh (21908) user_info::create: user shared version AB1FCCE8
  513   97747 [main] ssh (21908) fhandler_pipe::create: name \\.\pipe\msys-dd50a72ab4668b33-21908-sigwait, size 11440, mode PIPE_TYPE_MESSAGE
  503   98250 [main] ssh (21908) fhandler_pipe::create: pipe read handle 0xA0
  386   98636 [main] ssh (21908) fhandler_pipe::create: CreateFile: name \\.\pipe\msys-dd50a72ab4668b33-21908-sigwait
  515   99151 [main] ssh (21908) fhandler_pipe::create: pipe write handle 0xA4
 1380  100531 [main] ssh (21908) dll_crt0_0: finished dll_crt0_0 initialization
--- Process 21908 loaded C:\Program Files\Avecto\Privilege Guard Client\PGHook.dll at 000007fefc6f0000
--- Process 21908 loaded C:\Windows\System32\api-ms-win-core-synch-l1-2-0.dll at 000007fefc640000
--- Process 21908 loaded C:\Windows\System32\cryptsp.dll at 000007fefc620000
--- Process 21908 loaded C:\Windows\System32\rsaenh.dll at 000007fefc5d0000
--- Process 21908 loaded C:\Windows\System32\shell32.dll at 000007fefcec0000
--- Process 21908 loaded C:\Windows\System32\shlwapi.dll at 000007fefefa0000
--- Process 21908 loaded C:\Windows\System32\gdi32.dll at 000007fefe830000
--- Process 21908 loaded C:\Windows\System32\user32.dll at 0000000076d10000
--- Process 21908 loaded C:\Windows\System32\lpk.dll at 000007fefce90000
--- Process 21908 loaded C:\Windows\System32\usp10.dll at 000007fefea80000
--- Process 21908 loaded C:\Windows\System32\imm32.dll at 000007fefe940000
--- Process 21908 loaded C:\Windows\System32\msctf.dll at 000007fefe970000
--- Process 21908 loaded C:\Windows\System32\lsihok64.dll at 00000000745d0000
--- Process 21908 thread 21936 created
--- Process 21908, exception c0000005 at 000000018013c104
--- Process 21908 loaded C:\Windows\System32\version.dll at 000007fefc5b0000
--- Process 21908 unloaded DLL at 000007fefc5b0000
--- Process 21908 thread 21936 exited with status 0x0
--- Process 21908 exited with status 0xc0000005
Segmentation fault

I'm was thinking this might have had something to do with the PGHook.dll thats loaded by Avecto Privilege Guard, but the version of ssh & msys2 shipped with git-bash 2.17.1 does work, the latest one has the same problem as regular msys2. It still segfaults in strace for some reason.
That has ssh -V: OpenSSH_7.7p1, OpenSSL 1.0.2o 27 Mar 2018
While the one that fails is: OpenSSH_8.0p1, OpenSSL 1.1.1c 28 May 2019

strace ssh sdf.org on git-bash 2.17:

--- Process 22748 created
--- Process 22748 loaded C:\Windows\System32\ntdll.dll at 0000000076e10000
--- Process 22748 loaded C:\Windows\System32\kernel32.dll at 0000000076bf0000
--- Process 22748 loaded C:\Windows\System32\KernelBase.dll at 000007fefcb60000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-crypto-1.0.0.dll at 000000006f880000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-2.0.dll at 0000000180040000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-z.dll at 000000006abc0000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-gssapi-3.dll at 000000006e9b0000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-asn1-8.dll at 000000006fbd0000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-com_err-1.dll at 000000006fae0000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-roken-18.dll at 000000006b970000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-crypt-0.dll at 000000006fab0000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-heimntlm-0.dll at 000000006e7f0000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-krb5-26.dll at 000000006c510000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-heimbase-1.dll at 000000006e800000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-wind-0.dll at 000000006ad60000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-hx509-5.dll at 000000006e7a0000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-hcrypto-4.dll at 000000006e850000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-gcc_s-seh-1.dll at 000000006f330000
--- Process 22748 loaded C:\Program Files\Git\usr\bin\msys-sqlite3-0.dll at 000000006b800000
    3       3 [main] ssh (22748) **********************************************
  261     264 [main] ssh (22748) Program name: C:\Program Files\Git\usr\bin\ssh.exe (windows pid 22748)
  487     751 [main] ssh (22748) OS version:   Windows NT-6.1
  444    1195 [main] ssh (22748) **********************************************
--- Process 22748 loaded C:\Windows\System32\advapi32.dll at 000007fefe6b0000
--- Process 22748 loaded C:\Windows\System32\msvcrt.dll at 000007fefed70000
--- Process 22748 loaded C:\Windows\System32\sechost.dll at 000007fefeb50000
--- Process 22748 loaded C:\Windows\System32\rpcrt4.dll at 000007fefe450000
--- Process 22748 loaded C:\Windows\System32\cryptbase.dll at 000007fefc5c0000
79060   80255 [main] ssh (22748) sigprocmask: 0 = sigprocmask (0, 0x0, 0x1802F6C50)
  558   80813 [main] ssh 22748 open_shared: name shared.5, n 5, shared 0x180030000 (wanted 0x180030000), h 0x88, *m 6
  224   81037 [main] ssh 22748 user_heap_info::init: heap base 0x600000000, heap top 0x600000000, heap size 0x20000000 (536870912)
  472   81509 [main] ssh 22748 open_shared: name S-1-5-21-1796241753-961618004-8547516-351679.1, n 1, shared 0x180020000 (wanted 0x180020000), h 0x8C, *m 6
  457   81966 [main] ssh 22748 user_info::create: opening user shared for 'S-1-5-21-1796241753-961618004-8547516-351679' at 0x180020000
  467   82433 [main] ssh 22748 user_info::create: user shared version AB1FCCE8
  473   82906 [main] ssh 22748 fhandler_pipe::create: name \\.\pipe\msys-1888ae32e00d56aa-22748-sigwait, size 11440, mode PIPE_TYPE_MESSAGE
  516   83422 [main] ssh 22748 fhandler_pipe::create: pipe read handle 0xA0
  361   83783 [main] ssh 22748 fhandler_pipe::create: CreateFile: name \\.\pipe\msys-1888ae32e00d56aa-22748-sigwait
  537   84320 [main] ssh 22748 fhandler_pipe::create: pipe write handle 0xA4
  438   84758 [main] ssh 22748 dll_crt0_0: finished dll_crt0_0 initialization
--- Process 22748 loaded C:\Program Files\Avecto\Privilege Guard Client\PGHook.dll at 000007fefc6f0000
--- Process 22748 loaded C:\Windows\System32\api-ms-win-core-synch-l1-2-0.dll at 000007fefc640000
--- Process 22748 loaded C:\Windows\System32\cryptsp.dll at 000007fefc620000
--- Process 22748 loaded C:\Windows\System32\rsaenh.dll at 000007fefc5d0000
--- Process 22748 loaded C:\Windows\System32\shell32.dll at 000007fefcec0000
--- Process 22748 loaded C:\Windows\System32\shlwapi.dll at 000007fefefa0000
--- Process 22748 loaded C:\Windows\System32\gdi32.dll at 000007fefe830000
--- Process 22748 loaded C:\Windows\System32\user32.dll at 0000000076d10000
--- Process 22748 loaded C:\Windows\System32\lpk.dll at 000007fefce90000
--- Process 22748 loaded C:\Windows\System32\usp10.dll at 000007fefea80000
--- Process 22748 loaded C:\Windows\System32\imm32.dll at 000007fefe940000
--- Process 22748 loaded C:\Windows\System32\msctf.dll at 000007fefe970000
--- Process 22748 loaded C:\Windows\System32\lsihok64.dll at 00000000745d0000
--- Process 22748, exception c0000005 at 000000018012dc84
--- Process 22748 thread 20572 created
--- Process 22748 thread 23564 exited with status 0xc0000005
--- Process 22748 exited with status 0xc0000005
Segmentation fault

A difference I see is that on git-bash 2.17 it does not say create_child: ssh sdf.org, so did some behaviour change for openssh 8 which added forking or something similar?

Is there a way to fix this or will I have to stick with a separate install of git-bash 2.17 for all things ssh?

[Northcode]

After poking around the disassembly with snowman (https://derevenets.com/), it looks like the code at that address might be in this file: https://github.com/Alexpux/Cygwin/blob/msys2-master/winsup/cygwin/shm.cc as the code around the exception address seems to match up with the functions in that file, theres a string "shmdt (shmaddr = %p)" that matches the printf in shmdt() for example.

The function that causes the exception also seems pretty short, the decompiled version snowman managed to make is here:

int32_t fun_18012b5d5() {
    int32_t v1;
    void** r10_2;
    int32_t rax3;
    int32_t tmp32_4;
    void** v5;

    v1 = reinterpret_cast<int32_t>(__return_address());
    r10_2 = *reinterpret_cast<void***>(reinterpret_cast<int64_t>(&__real__Znwm) + 8);
    if (reinterpret_cast<int64_t>(__zero_stack_offset()) >= reinterpret_cast<int64_t>(r10_2 + 0xffffffffffffec58) || *reinterpret_cast<int32_t*>(r10_2 + 0xffffffffffffec58) != 0xc763173f) {
        return rax3;
    } else {
        while (tmp32_4 = *reinterpret_cast<int32_t*>(r10_2 + 0xffffffffffffe44c), *reinterpret_cast<int32_t*>(r10_2 + 0xffffffffffffe44c) = 1, *reinterpret_cast<int32_t*>(r10_2 + 0xffffffffffffe448) = tmp32_4, !!tmp32_4) {
            __asm__("pause ");
        }
        __asm__("xadd [r10-0x1bb0], rax");
        *reinterpret_cast<void***>(reinterpret_cast<int64_t>(&__real__Znwm) + 8) = v5;
        *reinterpret_cast<int32_t*>(r10_2 + 0xffffffffffffe444) = *reinterpret_cast<int32_t*>(r10_2 + 0xffffffffffffe444) + 1;
        *reinterpret_cast<int32_t*>(r10_2 + 0xffffffffffffe44c) = *reinterpret_cast<int32_t*>(r10_2 + 0xffffffffffffe44c) - 1;
        return v1;
    }
}

looks like some sort of spinlock, is it a mutex guard maybe?

The exception happens at the *reinterpret_cast<void***>(reinterpret_cast<int64_t>(&__real__Znwm) + 8) = v5; line.

[dalethatcher]

Does it fail in the same place when running ssh-keygen, as per this issue: #1731

Might be a smaller reproduction case.

[Northcode]

I forgot this was here.
I no longer use a windows machine. But last I remember this worked with the latest version of openssh.

closing this since I'm unable to follow up.

[darinkes]

I would like to reopen this Issue since it appeared in a setup.

The git-for-windows ssh-client crashes with
Exception thrown at 0x00000002101B5FE4 (msys-2.0.dll) in ssh.exe: 0xC0000005: Access violation writing location 0x0000000800300000.
when connecting to another Windows Server with OpenSSH-Server.

It happens as soon as the shell request is accepted.

debug2: shell request accepted on channel 0
=> dead

All I do to trigger or fix it is to switch between git-for-windows Version Git-2.28.0-64-bit and Git-2.44.0-64-bit.
Git-2.28.0-64-bit works fine, Git-2.44.0-64-bit crashes when connecting to Windows Servers with SSH.

The systems ssh-client installed by Microsofts OpenSSH-Port works fine.