Fix Unicorn Engine 1GB limit that calls exit: raise OSError instead (Fixes #2343)

[disconnect3d]

This commit fixes #2343: an issue where pwn checksec <binary> would fail with a bogus error of:

$ pwn checksec /root/x/bin/main
Could not allocate dynamic translator buffer

This actually comes from Unicorn Engine which tries to allocate a 1GB RWX mapping:

root@pwndbg:~# strace -e openat,mmap pwn checksec /root/x/bin/main 2>&1 | tail -n 10
openat(AT_FDCWD, "/usr/lib/python3/dist-packages/mpmath-0.0.0.egg-info/PKG-INFO", O_RDONLY|O_CLOEXEC) = 7
openat(AT_FDCWD, "/usr/local/lib/python3.10/dist-packages/unicorn/lib/libunicorn.so.2", O_RDONLY|O_CLOEXEC) = 7
mmap(NULL, 22447520, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 7, 0) = 0x7f2604f9d000
mmap(0x7f2605339000, 13496320, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0x39c000) = 0x7f2605339000
mmap(0x7f2606018000, 3039232, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0x107b000) = 0x7f2606018000
mmap(0x7f26062fe000, 1601536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 7, 0x1360000) = 0x7f26062fe000
mmap(0x7f2606485000, 525728, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f2606485000
mmap(NULL, 1073741824, PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = -1 ENOMEM (Cannot allocate memory)
Could not allocate dynamic translator buffer
+++ exited with 1 +++

...and if it fails, it calls exit(). This can be seen in Unicorn Engine code: https://github.com/unicorn-engine/unicorn/blob/56f3bdedb42d26bee1532cc01baf5eaf44a9aa23/qemu/accel/tcg/translate-all.c#L960-L963

This issue has been reported to Unicorn Engine in unicorn-engine/unicorn#1766 but since it hasn't been fixed, this commit applies a workaround for it.

[disconnect3d]

Fwiw we do a similar checking in https://github.com/pwndbg/pwndbg/blob/6b58f90bd8457c42fea0547b3e9d835650fdf6f5/pwndbg/disasm/__init__.py#L224-L255

[peace-maker]

Ugh, that's an annoying error handler. Thank you!

I've ran into a similar problem when using the github cli after running your ulimit -v 500000 test. So I'd rather set the memory limit back in CI too.

$ gh pr list
fatal error: failed to reserve page summary memory

runtime stack:
runtime.throw({0x15890b9?, 0x7f27b7877008?})
        /opt/hostedtoolcache/go/1.21.6/x64/src/runtime/panic.go:1077 +0x5c fp=0x7fff6c876518 sp=0x7fff6c8764e8 pc=0x438ffc
runtime.(*pageAlloc).sysInit(0x2d954e8, 0xb8?)
        /opt/hostedtoolcache/go/1.21.6/x64/src/runtime/mpagealloc_64bit.go:81 +0x11c fp=0x7fff6c876588 sp=0x7fff6c876518 pc=0x42fa7c
runtime.(*pageAlloc).init(0x2d954e8, 0x2d954e0, 0x2db1cb8, 0x0)
        /opt/hostedtoolcache/go/1.21.6/x64/src/runtime/mpagealloc.go:320 +0x85 fp=0x7fff6c8765b8 sp=0x7fff6c876588 pc=0x42d465
runtime.(*mheap).init(0x2d954e0)
        /opt/hostedtoolcache/go/1.21.6/x64/src/runtime/mheap.go:779 +0x16e fp=0x7fff6c876620 sp=0x7fff6c8765b8 pc=0x42a90e
runtime.mallocinit()
        /opt/hostedtoolcache/go/1.21.6/x64/src/runtime/malloc.go:435 +0xb2 fp=0x7fff6c876658 sp=0x7fff6c876620 pc=0x40ef92
runtime.schedinit()
        /opt/hostedtoolcache/go/1.21.6/x64/src/runtime/proc.go:726 +0x93 fp=0x7fff6c8766a0 sp=0x7fff6c876658 pc=0x43cad3
runtime.rt0_go()
        /opt/hostedtoolcache/go/1.21.6/x64/src/runtime/asm_amd64.s:349 +0x11c fp=0x7fff6c8766a8 sp=0x7fff6c8766a0 pc=0x46993c