Fix Issue 12164 - Function returning ptrdiff_t.min in 64-bit returnin…

…g 0 when -O is set

The trick used to optimize ?const:const doesn't work when you can't actually express the value as immediates.  They were silently truncated leading to incorrect results.

Values were also truncated due to a typo.

src/backend/cod2.c

@@ -1910,21 +1910,18 @@ code *cdcond(elem *e,regm_t *pretregs)
         e22->Eoper == OPconst
      )
   {     regm_t retregs;
-        unsigned reg;
         targ_size_t v1,v2;
         int opcode;
 
         retregs = *pretregs & (ALLREGS | mBP);
         if (!retregs)
             retregs = ALLREGS;
         cdcmp_flag = 1;
-        c = codelem(e1,&retregs,FALSE);
-        reg = findreg(retregs);
         v1 = e21->EV.Vllong;
         v2 = e22->EV.Vllong;
         if (jop == JNC)
         {   v1 = v2;
-            v2 = e21->EV.Vlong;
+            v2 = e21->EV.Vllong;
         }
 
         opcode = 0x81;
@@ -1941,35 +1938,49 @@ code *cdcond(elem *e,regm_t *pretregs)
                         break;
         }
 
-        if (v1 == 0 && v2 == ~(targ_size_t)0)
+        if (I64 && v1 != (targ_ullong)(targ_ulong)v1)
         {
-            c = gen2(c,0xF6 + (opcode & 1),grex | modregrmx(3,2,reg));  // NOT reg
-            if (I64 && sz2 == REGSIZE)
-                code_orrex(c, REX_W);
+            // only zero-extension from 32-bits is available for 'or'
+        }
+        else if (I64 && v2 != (targ_llong)(targ_long)v2)
+        {
+            // only sign-extension from 32-bits is available for 'and'
         }
         else
         {
-            v1 -= v2;
-            c = genc2(c,opcode,grex | modregrmx(3,4,reg),v1);   // AND reg,v1-v2
-            if (I64 && sz1 == 1 && reg >= 4)
-                code_orrex(c, REX);
-            if (v2 == 1 && !I64)
-                gen1(c,0x40 + reg);                     // INC reg
-            else if (v2 == -1L && !I64)
-                gen1(c,0x48 + reg);                     // DEC reg
+            c = codelem(e1,&retregs,FALSE);
+            unsigned reg = findreg(retregs);
+
+            if (v1 == 0 && v2 == ~(targ_size_t)0)
+            {
+                c = gen2(c,0xF6 + (opcode & 1),grex | modregrmx(3,2,reg));  // NOT reg
+                if (I64 && sz2 == REGSIZE)
+                    code_orrex(c, REX_W);
+            }
             else
-            {   genc2(c,opcode,grex | modregrmx(3,0,reg),v2);   // ADD reg,v2
+            {
+                v1 -= v2;
+                c = genc2(c,opcode,grex | modregrmx(3,4,reg),v1);   // AND reg,v1-v2
                 if (I64 && sz1 == 1 && reg >= 4)
                     code_orrex(c, REX);
+                if (v2 == 1 && !I64)
+                    gen1(c,0x40 + reg);                     // INC reg
+                else if (v2 == -1L && !I64)
+                    gen1(c,0x48 + reg);                     // DEC reg
+                else
+                {   genc2(c,opcode,grex | modregrmx(3,0,reg),v2);   // ADD reg,v2
+                    if (I64 && sz1 == 1 && reg >= 4)
+                        code_orrex(c, REX);
+                }
             }
-        }
 
-        freenode(e21);
-        freenode(e22);
-        freenode(e2);
+            freenode(e21);
+            freenode(e22);
+            freenode(e2);
 
-        c = cat(c,fixresult(e,retregs,pretregs));
-        goto Lret;
+            c = cat(c,fixresult(e,retregs,pretregs));
+            goto Lret;
+        }
   }
 
   if (op1 != OPcond && op1 != OPandand && op1 != OPoror &&

test/runnable/mars1.d

@@ -1034,6 +1034,27 @@ void test10715()
 
 ////////////////////////////////////////////////////////////////////////
 
+ptrdiff_t compare12164(A12164* rhsPA, A12164* zis)
+{
+    if (*rhsPA == *zis)
+        return 0;
+    return ptrdiff_t.min;
+}
+
+struct A12164
+{
+    int a;
+}
+
+void test12164()
+{
+    auto a = A12164(3);
+    auto b = A12164(2);
+    assert(compare12164(&a, &b));
+}
+
+////////////////////////////////////////////////////////////////////////
+
 int foo10678(char[5] txt)
 {
     return txt[0] + txt[1] + txt[4];
@@ -1147,6 +1168,7 @@ int main()
     testbreak();
     teststringswitch();
     teststrarg();
+    test12164();
     testsizes();
     testarrayinit();
     testU();