bswap r9w (undefined behavior/bad operand) needs to always be 0x00

[brandonros]

RUST_BACKTRACE=1 cargo run --target x86_64-apple-darwin --release -- -6 -vvv --regs --nocolors -f /Users/brandonros/Downloads/DTS9_PatcherV.exe > output.txt

{
    "message": "newValue mismatch",
    "i": 4,
    "x64dbgLine": {
      "rawLine": {
        "Index": "00004",
        "Address": "0000000144F84075",
        "Bytes": "6644:0FB6C8",
        "Disassembly": "movzx r9w,al",
        "Registers": "r9: 0-> A4",
        "Memory": "",
        "Comments": ""
      },
      "rip": "144f84075",
      "registerChanges": [
        {
          "registerName": "r9",
          "previousValue": "0",
          "newValue": "a4"
        }
      ],
      "memoryChanges": [
        ""
      ]
    },
    "scemuLine": {
      "rip": "144f84075",
      "registerChanges": [
        {
          "registerName": "r9",
          "previousValue": "0",
          "newValue": "8a"
        }
      ],
      "memoryChanges": []
    }
  },

[brandonros]

so I did added this:

    pub fn init_stack64(&mut self) {
        let stack = self.maps.get_mem("stack");

        // regular
        //self.regs.rsp = 0x22e000;
        //self.regs.rbp = 0x22f000;
        //stack.set_base(0x22a000);
        //stack.set_size(0x6000);

        // DTS9_PatcherV.exe
        self.regs.rsp = 0x000000000014F4B0;
        self.regs.rbp = 0x0000000000000000;
        stack.set_base(0x0000000000149000);
        stack.set_size(0x0000000000007000);

        //assert!(self.regs.rsp < self.regs.rbp);
        //assert!(self.regs.rsp > stack.get_base());
        //assert!(self.regs.rsp < stack.get_bottom());
        //assert!(self.regs.rbp > stack.get_base());
        //assert!(self.regs.rbp < stack.get_bottom());
        //assert!(stack.inside(self.regs.rsp));
        //assert!(stack.inside(self.regs.rbp));
    }

    pub fn init_registers64(&mut self) {
        // set all to 0
        self.regs.rax = 0x0000000000000000;
        self.regs.rbx = 0x0000000000000000;
        self.regs.rcx = 0x0000000000000000;
        self.regs.rdx = 0x0000000000000000;
        self.regs.rsp = 0x0000000000000000;
        self.regs.rbp = 0x0000000000000000;
        self.regs.rsi = 0x0000000000000000;
        self.regs.rdi = 0x0000000000000000;
        self.regs.r8 = 0x0000000000000000;
        self.regs.r9 = 0x0000000000000000;
        self.regs.r10 = 0x0000000000000000;
        self.regs.r11 = 0x0000000000000000;
        self.regs.r12 = 0x0000000000000000;
        self.regs.r13 = 0x0000000000000000;
        self.regs.r14 = 0x0000000000000000;
        self.regs.r15 = 0x0000000000000000;
        // DTS9_PatcherV.exe
        self.regs.rax = 0x00000001448A76A4;
        self.regs.rbx = 0x000000007FFE0385;
        self.regs.rcx = 0x0000000140000000;
        self.regs.rdx = 0x0000000000000001;
        self.regs.rsi = 0x0000000000000001;
        self.regs.rdi = 0x000000007FFE0384;
        self.regs.r10 = 0x000000007FFE0384;
        self.regs.r11 = 0x0000000000000246;
        self.regs.r12 = 0x00000001448A76A4;
        self.regs.r14 = 0x0000000140000000;
    }

[brandonros]

gets us to this

{
    "message": "newValue mismatch",
    "i": 12,
    "x64dbgLine": {
      "rawLine": {
        "Index": "0000C",
        "Address": "0000000144ED49FE",
        "Bytes": "6641:0FC9",
        "Disassembly": "bswap r9w",
        "Registers": "r9: 84-> 0",
        "Memory": "",
        "Comments": ""
      },
      "rip": "144ed49fe",
      "registerChanges": [
        {
          "registerName": "r9",
          "previousValue": "84",
          "newValue": "0"
        }
      ],
      "memoryChanges": [
        ""
      ]
    },
    "scemuLine": {
      "rip": "144ed49fe",
      "registerChanges": [
        {
          "registerName": "r9",
          "previousValue": "84",
          "newValue": "8400"
        }
      ],
      "memoryChanges": []
    }
  },

[brandonros]

sounds like we have a bswap bug with r9w style register (8400 vs 84)

[sha0coder]

bswap of 16bits has undefined behaviour on a processor, xchg has to be used instead regarding the manual.

the current implementation is this, but it seem the processor is keeping the value, i will change it.

} else if sz == 16 {
                    value1 = (value0 & 0x00000000_000000ff) << 8 | (value0 & 0x00000000_0000ff00) >> 8;

[sha0coder]

it seems the packer is using multiple instructions with undefined behaviors but they might not affect the execution flow.

[sha0coder]

done,
there are more cases of undefined behaviours, for example shifting with counter > regsize.
will not match with x64dbg in those cases.

[brandonros]

{
    "message": "number of register changes mismatch",
    "i": 12,
    "x64dbgLine": {
      "rawLine": {
        "Index": "0000C",
        "Address": "0000000144ED49FE",
        "Bytes": "6641:0FC9",
        "Disassembly": "bswap r9w",
        "Registers": "r9: 84-> 0",
        "Memory": "",
        "Comments": ""
      },
      "rip": "144ed49fe",
      "registerChanges": [
        {
          "registerName": "r9",
          "previousValue": "84",
          "newValue": "0"
        }
      ],
      "memoryChanges": [
        ""
      ]
    },
    "scemuLine": {
      "rawLine": "diff_reg: rip = 144ed49fe",
      "rip": "144ed49fe",
      "registerChanges": [],
      "memoryChanges": []
    }
  },

different error on the same register

[sha0coder]

I would skip the 16bits bswap because processor's result is not predictable.

[brandonros]

did you add the inline test that validates whether the emulation result matches what the processor would do for bswap? i don't think i can skip it because it throws the rest of the stuff off / the packer is relying on the "undefined operation" to be predictable

[sha0coder]

the manual says undefined behavior, but we could try to find the logic and implement it.
do you think that the result of this instruction can influence on the execution flow?
the compiler or packer that generated it also don't know that logic.

[sha0coder]

the linine test is checking 32 and 64 bits bswap only and matches.

[sha0coder]

like on opaque predicates there are unreachable instructions or reachable but that don't influence on the result. It could be that.

[sha0coder]

a weird thing on inline::div

            unsafe {   
                asm!("int 3");
                asm!("mov rax, {}", in(reg) rax);
                asm!("mov rdx, {}", in(reg) rdx);
                asm!("div {:e}", in(reg) a32);
                asm!("mov {}, rdx", out(reg) r_rdx);
                asm!("mov {}, rax", out(reg) r_rax);
            }  

last two mov's overlap rdx, i lost the result on rdx

Thread 1 "scemu" received signal SIGTRAP, Trace/breakpoint trap.
0x0000555555670f07 in scemu::emu::inline::div ()
(gdb) x/i $pc
=> 0x555555670f07 <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+55>:    mov    %rsi,%rax
(gdb) 
   0x555555670f0a <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+58>:    mov    %rdx,%rdx
(gdb) 
   0x555555670f0d <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+61>:    div    %edi
(gdb) 
   0x555555670f0f <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+63>:    mov    %rdx,%rax
(gdb) 
   0x555555670f12 <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+66>:    mov    %rax,%rdx

in C I used to do volatile to keep the registers.

[sha0coder]

there are problems to extracting eax

          unsafe {   
                asm!("int 3");  
                asm!("mov rax, {}", in(reg) rax);  
                asm!("mov rdx, {}", in(reg) rdx);  
                asm!("div {:e}", in(reg) a32);  
                asm!("mov rsi, rax", out("rsi") r_rax);  
                asm!("mov rdi, rdx", out("rdi") r_rdx);   
            } 

Thread 1 "scemu" received signal SIGTRAP, Trace/breakpoint trap.
0x0000555555670f07 in scemu::emu::inline::div ()
(gdb) x/i $pc
=> 0x555555670f07 <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+55>:    mov    %rsi,%rax
(gdb) 
   0x555555670f0a <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+58>:    mov    %rdx,%rdx
(gdb) 
   0x555555670f0d <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+61>:    div    %edi
(gdb) 
   0x555555670f0f <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+63>:    mov    %rax,%rsi   Ok extract rax
(gdb) 
   0x555555670f12 <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+66>:    mov    %rsi,%rdx  What!! altering rdx??
(gdb) 
   0x555555670f15 <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+69>:    mov    %rdx,%rdi  Extracting altered rdx
(gdb) 
   0x555555670f18 <_ZN5scemu3emu6inline3div17hf3b14b002d24226fE+72>:    mov    %rdi,%rax  Altering rax

[sha0coder]

fixed including both mov in one asm!()

 unsafe {   
                asm!("mov rax, {}", in(reg) rax);  
                asm!("mov rdx, {}", in(reg) rdx);  
                asm!("div {:e}", in(reg) a32);  
                asm!("mov rsi, rax", "mov rdi, rdx", out("rsi") r_rax, out("rdi") r_rdx);
            }