i#58 MacOS: handle system call invocation features unique to MacOS

1) Encode syscall numbers properly for MacOS: high-order bits are used to
   pass the system call class for 64-bit, and the argument size for 32-bit.

2) 32-bit syscall args are on the stack and not in registers

3) Syscall failure is indicated by setting CF and returning +errno, rather
   than by returning -errno.

These are fixed for DR's invocation of its own syscalls.

Still TODO: handling all 3 of these for monitoring app syscalls.

SVN-Revision: 2456

core/x86/x86.asm

@@ -1080,6 +1080,10 @@ GLOBAL_LABEL(dynamorio_syscall:)
         /* reverse order so we don't clobber earlier args */
         mov      REG_XBX, ARG2 /* put num_args where we can reference it longer */
         mov      rax, ARG1 /* sysnum: only need eax, but need rax to use ARG1 (or movzx) */
+#  ifdef MACOS
+        /* for now we assume a BSD syscall */
+        or       rax, 0x2000000
+#   endif
         cmp      REG_XBX, 0
         je       syscall_ready
         mov      ARG1, ARG3
@@ -1134,16 +1138,42 @@ syscall_2args:
 syscall_1args:
         mov      ebx, [16+12 + esp] /* arg1 */
 syscall_0args:
+#  ifdef MACOS
+        /* Arg size is encoded in upper bits.
+         * XXX: or is that only for sysenter gateway?
+         */
+        mov      eax, [16+ 8 + esp] /* num_args */
+        shl      eax, 18 /* <<16 but also *4 for size */
+        or       eax, [16+ 4 + esp] /* sysnum */
+        /* args are on stack, w/ an extra slot (retaddr of syscall wrapper) */
+        push     ebp
+        push     edi
+        push     esi
+        push     edx
+        push     ecx
+        push     ebx
+        push     0 /* extra slot */
+#  else
         mov      eax, [16+ 4 + esp] /* sysnum */
+#  endif
         /* PR 254280: we assume int$80 is ok even for LOL64 */
         int      HEX(80)
+#  ifdef MACOS
+        lea      esp, [7*ARG_SZ + esp] /* must not change flags */
+#  endif
         pop      REG_XDI
         pop      REG_XSI
         pop      REG_XBP
 # endif /* X64 */
         pop      REG_XBX
         /* return val is in eax for us */
         /* for MacOS, it can also include edx, so be sure not to clobber that! */
+# ifdef MACOS
+        /* convert to -errno */
+        jae      syscall_success
+        neg      eax
+syscall_success:
+# endif
         ret
         END_FUNC(dynamorio_syscall)