From 237d45700eef31e23e8d10459335501f87540513 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caol=C3=A1n=20McNamara?= <caolanm@redhat.com>
Date: Mon, 15 May 2017 11:17:57 +0100
Subject: [PATCH] ofz#1605 check multiply and shift
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Change-Id: I6aad9ad23e7bf080b3b610223f92df7074530beb
Reviewed-on: https://gerrit.libreoffice.org/37632
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>
---
 vcl/headless/svpbmp.cxx        | 18 ++++++++++++++++--
 vcl/source/gdi/salmisc.cxx     | 19 ++++++++++++++++++-
 vcl/unx/generic/gdi/salbmp.cxx | 17 ++++++++++++++++-
 3 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/vcl/headless/svpbmp.cxx b/vcl/headless/svpbmp.cxx
index 3091dfba888ce..5be976fc9e748 100644
--- a/vcl/headless/svpbmp.cxx
+++ b/vcl/headless/svpbmp.cxx
@@ -27,7 +27,7 @@
 
 #include <basegfx/vector/b2ivector.hxx>
 #include <basegfx/range/b2ibox.hxx>
-
+#include <o3tl/safeint.hxx>
 #include <vcl/salbtype.hxx>
 #include <vcl/bitmap.hxx>
 
@@ -112,7 +112,21 @@ BitmapBuffer* ImplCreateDIB(
     pDIB->mnFormat |= ScanlineFormat::TopDown;
     pDIB->mnWidth = rSize.Width();
     pDIB->mnHeight = rSize.Height();
-    pDIB->mnScanlineSize = AlignedWidth4Bytes( pDIB->mnWidth * nBitCount );
+    long nScanlineBase;
+    bool bFail = o3tl::checked_multiply<long>(pDIB->mnWidth, nBitCount, nScanlineBase);
+    if (bFail)
+    {
+        SAL_WARN("vcl.gdi", "checked multiply failed");
+        delete pDIB;
+        return nullptr;
+    }
+    pDIB->mnScanlineSize = AlignedWidth4Bytes(nScanlineBase);
+    if (pDIB->mnScanlineSize < nScanlineBase/8)
+    {
+        SAL_WARN("vcl.gdi", "scanline calculation wraparound");
+        delete pDIB;
+        return nullptr;
+    }
     pDIB->mnBitCount = nBitCount;
 
     if( nColors )
diff --git a/vcl/source/gdi/salmisc.cxx b/vcl/source/gdi/salmisc.cxx
index ffde759517fa6..4f239efb95c6e 100644
--- a/vcl/source/gdi/salmisc.cxx
+++ b/vcl/source/gdi/salmisc.cxx
@@ -20,6 +20,7 @@
 #include <vcl/bitmapaccess.hxx>
 #include <vcl/salbtype.hxx>
 #include <bmpfast.hxx>
+#include <o3tl/safeint.hxx>
 #include <osl/diagnose.h>
 #include <memory>
 
@@ -328,7 +329,23 @@ BitmapBuffer* StretchAndConvert(
     pDstBuffer->mnFormat = nDstBitmapFormat;
     pDstBuffer->mnWidth = rTwoRect.mnDestWidth;
     pDstBuffer->mnHeight = rTwoRect.mnDestHeight;
-    pDstBuffer->mnScanlineSize = AlignedWidth4Bytes( pDstBuffer->mnBitCount * pDstBuffer->mnWidth );
+    long nScanlineBase;
+    bool bFail = o3tl::checked_multiply<long>(pDstBuffer->mnBitCount, pDstBuffer->mnWidth, nScanlineBase);
+    if (bFail)
+    {
+        SAL_WARN("vcl.gdi", "checked multiply failed");
+        pDstBuffer->mpBits = nullptr;
+        delete pDstBuffer;
+        return nullptr;
+    }
+    pDstBuffer->mnScanlineSize = AlignedWidth4Bytes(nScanlineBase);
+    if (pDstBuffer->mnScanlineSize < nScanlineBase/8)
+    {
+        SAL_WARN("vcl.gdi", "scanline calculation wraparound");
+        pDstBuffer->mpBits = nullptr;
+        delete pDstBuffer;
+        return nullptr;
+    }
     try
     {
         pDstBuffer->mpBits = new sal_uInt8[ pDstBuffer->mnScanlineSize * pDstBuffer->mnHeight ];
diff --git a/vcl/unx/generic/gdi/salbmp.cxx b/vcl/unx/generic/gdi/salbmp.cxx
index 73fc5a4241522..00043d5e783e8 100644
--- a/vcl/unx/generic/gdi/salbmp.cxx
+++ b/vcl/unx/generic/gdi/salbmp.cxx
@@ -41,6 +41,7 @@
 #include <unx/salinst.h>
 #include <unx/x11/xlimits.hxx>
 
+#include <o3tl/safeint.hxx>
 #include <opengl/salbmp.hxx>
 #include <vcl/opengl/OpenGLHelper.hxx>
 
@@ -193,7 +194,21 @@ BitmapBuffer* X11SalBitmap::ImplCreateDIB(
 
             pDIB->mnWidth = rSize.Width();
             pDIB->mnHeight = rSize.Height();
-            pDIB->mnScanlineSize = AlignedWidth4Bytes( pDIB->mnWidth * nBitCount );
+            long nScanlineBase;
+            bool bFail = o3tl::checked_multiply<long>(pDIB->mnWidth, nBitCount, nScanlineBase);
+            if (bFail)
+            {
+                SAL_WARN("vcl.gdi", "checked multiply failed");
+                delete pDIB;
+                return nullptr;
+            }
+            pDIB->mnScanlineSize = AlignedWidth4Bytes(nScanlineBase);
+            if (pDIB->mnScanlineSize < nScanlineBase/8)
+            {
+                SAL_WARN("vcl.gdi", "scanline calculation wraparound");
+                delete pDIB;
+                return nullptr;
+            }
             pDIB->mnBitCount = nBitCount;
 
             if( nColors )