AddressSanitizer: fix allocating too big buffers (libjpeg-turbo)

Fixex attempt to allocate 0xffffffff833db0fe bytes of data:

==18628==WARNING: AddressSanitizer failed to allocate 0xffffffff833db0fe bytes
==18628==AddressSanitizer CHECK failed: ../../../../libsanitizer/sanitizer_common/sanitizer_allocator.cc:218 "((0)) != (0)" (0x0, 0x0)
    #0 0x7f939d46a902  (/lib64/libasan.so.4+0xe9902)
    #1 0x7f939d489295 in __sanitizer::CheckFailed(char const*, int, char const*, unsigned long long, unsigned long long) (/lib64/libasan.so.4+0x108295)
    #2 0x7f939d470042  (/lib64/libasan.so.4+0xef042)
    #3 0x7f939d3a8b46  (/lib64/libasan.so.4+0x27b46)
    #4 0x7f939d45f81a in malloc (/lib64/libasan.so.4+0xde81a)
    #5 0x81fc71 in pxLoadJPGImageTurbo(char const*, unsigned long, pxOffscreen&) pxCore/src/pxUtil.cpp:711
    #6 0x81dea4 in pxLoadImage(char const*, unsigned long, pxOffscreen&) pxCore/src/pxUtil.cpp:49
    #7 0x63a942 in pxUtilTest::pxLoadImage3ArgsLessLengthFailureTest() pxCore/tests/pxScene2d/test_pxUtil.cpp:142
    #8 0x638975 in pxUtilTest_pxutilsTest_Test::TestBody() pxCore/tests/pxScene2d/test_pxUtil.cpp:327
    #9 0x7ad278 in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) (pxCore/tests/pxScene2d/pxscene2dtests+0x7ad278)
    #10 0x7a0938 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) pxCore/tests/pxScene2d/../../examples/pxScene2d/external/gtest/googletest/src/gtest.cc:2438
    #11 0x75d4cd in testing::Test::Run() pxCore/tests/pxScene2d/../../examples/pxScene2d/external/gtest/googletest/src/gtest.cc:2474
    #12 0x75e7b9 in testing::TestInfo::Run() pxCore/tests/pxScene2d/../../examples/pxScene2d/external/gtest/googletest/src/gtest.cc:2656
    #13 0x75f332 in testing::TestCase::Run() pxCore/tests/pxScene2d/../../examples/pxScene2d/external/gtest/googletest/src/gtest.cc:2774
    #14 0x76fed0 in testing::internal::UnitTestImpl::RunAllTests() pxCore/tests/pxScene2d/../../examples/pxScene2d/external/gtest/googletest/src/gtest.cc:4649
    #15 0x7afcb7 in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) pxCore/tests/pxScene2d/../../examples/pxScene2d/external/gtest/googletest/src/gtest.cc:2402
    #16 0x7a2a97 in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) (pxCore/tests/pxScene2d/pxscene2dtests+0x7a2a97)
    #17 0x76d0d9 in testing::UnitTest::Run() pxCore/tests/pxScene2d/../../examples/pxScene2d/external/gtest/googletest/src/gtest.cc:4257
    #18 0x5299db in RUN_ALL_TESTS() (pxCore/tests/pxScene2d/pxscene2dtests+0x5299db)
    #19 0x5295c7 in main pxCore/tests/pxScene2d/pxscene2dtestsmain.cpp:101
    #20 0x7f9397692009 in __libc_start_main (/lib64/libc.so.6+0x21009)
    #21 0x5293c9 in _start (pxCore/tests/pxScene2d/pxscene2dtests+0x5293c9)

src/pxUtil.cpp

@@ -87,8 +87,8 @@ rtError pxLoadAImage(const char* imageData, size_t imageDataSize,
     s.init();
     s.addBuffer(o,0);
   }
-  
-  return retVal;  
+
+  return retVal;
 }
 
 
@@ -359,7 +359,7 @@ rtError pxStorePNGImage(const char *filename, pxOffscreen &b, bool /*grayscale*/
  * conjunction with the documentation file libjpeg.txt.
  *
  * This code will not do anything useful as-is, but it may be helpful as a
- * skeleton for constructing routines that call the JPEG library.  
+ * skeleton for constructing routines that call the JPEG library.
  *
  * We present these routines in the same coding style used in the JPEG code
  * (ANSI function definitions, etc); but you are of course free to code your
@@ -708,8 +708,23 @@ rtError pxLoadJPGImageTurbo(const char *buf, size_t buflen, pxOffscreen &o)
     return RT_FAIL;// TODO : add grayscale support for libjpeg turbo.  falling back to libjpeg for now
   }
 
+  // limit memory usage to resolution 4096x4096
+  if (((size_t)width * height) > ((size_t)4096 * 4096))
+  {
+    rtLogError("Error libjpeg-turbo: image too large");
+    tjDestroy(jpegDecompressor);
+    return RT_FAIL;
+  }
+
   unsigned char *imageBuffer = tjAlloc(width * height * 3);
 
+  if (!imageBuffer)
+  {
+    rtLogError("Error allocating libjpeg-turbo buffer");
+    tjDestroy(jpegDecompressor);
+    return RT_FAIL;
+  }
+
   int result = tjDecompress2(jpegDecompressor, (unsigned char *)buf, buflen, imageBuffer, width, 0, height, TJPF_RGB /*(colorComponent == 3) ? TJPF_RGB : jpegColorspace*/, TJFLAG_FASTDCT);
 
   if (result != 0)
@@ -1151,9 +1166,9 @@ rtError pxLoadAPNGImage(const char *imageData, size_t imageDataSize,
 
   //unsigned int width, height, channels, rowbytes, size, i, j;
   unsigned int width, height, i, j;
-  
+
   unsigned long size, rowbytes;
-  
+
   png_bytepp rows_image;
   png_bytepp rows_frame;
   unsigned char *p_image;