add some allocation overflow checks

src/dmd/backend/barray.d

@@ -40,6 +40,7 @@ struct Barray(T)
         static void enlarge(ref Barray barray, size_t length)
         {
             pragma(inline, false);
+            assert(length < size_t.max / (2 * T.sizeof));       // conservative overflow check
             auto newcap = (barray.capacity == 0) ? length : length + (length >> 1);
             barray.capacity = (newcap + 15) & ~15;
             T* p = cast(T*)realloc(barray.array.ptr, barray.capacity * T.sizeof);

src/dmd/backend/cgelem.d

@@ -2889,6 +2889,7 @@ private bool optim_loglog(elem **pe)
     if (n <= 3)
         return false;
     uint ty = e.Ety;
+    assert(n < size_t.max / (2 * (elem *).sizeof));   // conservative overflow check
     elem **array = cast(elem **)malloc(n * (elem *).sizeof);
     assert(array);
     elem **p = array;

src/dmd/backend/cgxmm.d

@@ -1090,6 +1090,7 @@ void cdvector(ref CodeBuilder cdb, elem *e, regm_t *pretregs)
     }
 
     const n = el_nparams(e.EV.E1);
+    assert(n < size_t.max / (2 * (elem *).sizeof));   // conservative overflow check
     elem **params = cast(elem **)malloc(n * (elem *).sizeof);
     assert(params);
     elem **tmp = params;

src/dmd/backend/cod3.d

@@ -1744,6 +1744,7 @@ void doswitch(ref CodeBuilder cdb, block *b)
         reg_t sreg = NOREG;                          // may need a scratch register
 
         // Put into casevals[0..ncases] so we can sort then slice
+        assert(ncases < size_t.max / (2 * CaseVal.sizeof));
         CaseVal *casevals = cast(CaseVal *)malloc(ncases * CaseVal.sizeof);
         assert(casevals);
         for (uint n = 0; n < ncases; n++)

src/dmd/backend/dt.d

@@ -299,8 +299,10 @@ nothrow:
     void abytes(tym_t ty, uint offset, uint size, const(char)* ptr, uint nzeros, ubyte _align)
     {
         dt_t *dt = dt_calloc(DT_abytes);
-        dt.DTnbytes = size + nzeros;
-        dt.DTpbytes = cast(byte *) mem_malloc(size + nzeros);
+        const n = size + nzeros;
+        assert(n >= size);      // overflow check
+        dt.DTnbytes = n;
+        dt.DTpbytes = cast(byte *) mem_malloc(n);
         dt.Dty = cast(ubyte)ty;
         dt.DTalign = _align;
         dt.DTabytes = offset;
@@ -562,7 +564,9 @@ nothrow:
             return;
         }
 
-        char *p = cast(char *)mem_malloc(size * count);
+        const n = size * count;
+        assert(n >= size);
+        char *p = cast(char *)mem_malloc(n);
         size_t offset = 0;
 
         for (dt_t *dtn = dt; dtn; dtn = dtn.DTnext)

src/dmd/backend/dwarfeh.d

@@ -57,6 +57,7 @@ nothrow:
         assert(dim <= capacity);
         if (dim == capacity)
         {
+            assert(capacity < uint.max / (3 * DwEhTableEntry.sizeof));  // conservative overflow check
             capacity += capacity + 16;
             ptr = cast(DwEhTableEntry *)realloc(ptr, capacity * DwEhTableEntry.sizeof);
             assert(ptr);