Merge pull request #5397 from wiredfool/valgrind_fixes

Valgrind fixes

Makefile

@@ -102,6 +102,13 @@ sdist:
 test:
 	pytest -qq
 
+.PHONY: valgrind
+valgrind:
+	python3 -c "import pytest_valgrind" || pip3 install pytest-valgrind
+	PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp --leak-check=no \
+            --log-file=/tmp/valgrind-output \
+            python3 -m pytest --no-memcheck -vv --valgrind --valgrind-log=/tmp/valgrind-output
+
 .PHONY: readme
 readme:
 	python3 setup.py --long-description | markdown2 > .long-description.html && open .long-description.html

Tests/conftest.py

@@ -13,6 +13,11 @@ def pytest_report_header(config):
 
 
 def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "pil_noop_mark: A conditional mark where nothing special happens",
+    )
+
     # We're marking some tests to ignore valgrind errors and XFAIL them.
     # Ensure that the mark is defined
     # even in cases where pytest-valgrind isn't installed

Tests/helper.py

@@ -173,6 +173,21 @@ def skip_unless_feature_version(feature, version_required, reason=None):
     return pytest.mark.skipif(version_available < version_required, reason=reason)
 
 
+def mark_if_feature_version(mark, feature, version_blacklist, reason=None):
+    if not features.check(feature):
+        return pytest.mark.pil_noop_mark()
+    if reason is None:
+        reason = f"{feature} is {version_blacklist}"
+    version_required = parse_version(version_blacklist)
+    version_available = parse_version(features.version(feature))
+    if (
+        version_available.major == version_required.major
+        and version_available.minor == version_required.minor
+    ):
+        return mark(reason=reason)
+    return pytest.mark.pil_noop_mark()
+
+
 @pytest.mark.skipif(sys.platform.startswith("win32"), reason="Requires Unix or macOS")
 class PillowLeakTestCase:
     # requires unix/macOS

Tests/oss-fuzz/python.supp

@@ -0,0 +1,16 @@
+{
+   <py3_8_encode_current_locale>
+   Memcheck:Cond
+   ...
+   fun:encode_current_locale
+}
+
+
+{
+   <libtiff_zlib>
+   Memcheck:Cond
+   fun:inflate
+   fun:ZIPDecode
+   fun:_TIFFReadEncodedTileAndAllocBuffer
+   ...
+}

Tests/oss-fuzz/test_fuzzers.py

@@ -2,12 +2,19 @@
 import sys
 
 import fuzzers
+import packaging
 import pytest
 
-from PIL import Image
+from PIL import Image, features
 
 if sys.platform.startswith("win32"):
     pytest.skip("Fuzzer is linux only", allow_module_level=True)
+if features.check("libjpeg_turbo"):
+    version = packaging.version.parse(features.version("libjpeg_turbo"))
+    if version.major == 2 and version.minor == 0:
+        pytestmark = pytest.mark.valgrind_known_error(
+            reason="Known failing with libjpeg_turbo 2.0"
+        )
 
 
 @pytest.mark.parametrize(

Tests/test_file_eps.py

@@ -8,6 +8,7 @@
     assert_image_similar,
     assert_image_similar_tofile,
     hopper,
+    mark_if_feature_version,
     skip_unless_feature,
 )
 
@@ -64,7 +65,9 @@ def test_invalid_file():
         EpsImagePlugin.EpsImageFile(invalid_file)
 
 
-@pytest.mark.valgrind_known_error(reason="Known Failing")
+@mark_if_feature_version(
+    pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+)
 @pytest.mark.skipif(not HAS_GHOSTSCRIPT, reason="Ghostscript not available")
 def test_cmyk():
     with Image.open("Tests/images/pil_sample_cmyk.eps") as cmyk_image:

Tests/test_file_jpeg.py

@@ -24,6 +24,7 @@
     djpeg_available,
     hopper,
     is_win32,
+    mark_if_feature_version,
     skip_unless_feature,
 )
 
@@ -116,7 +117,9 @@ def test(xdpi, ydpi=None):
         assert test(100, 200) == (100, 200)
         assert test(0) is None  # square pixels
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_icc(self, tmp_path):
         # Test ICC support
         with Image.open("Tests/images/rgb.jpg") as im1:
@@ -156,7 +159,9 @@ def test(n):
         test(ImageFile.MAXBLOCK + 1)  # full buffer block plus one byte
         test(ImageFile.MAXBLOCK * 4 + 3)  # large block
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_large_icc_meta(self, tmp_path):
         # https://github.com/python-pillow/Pillow/issues/148
         # Sometimes the meta data on the icc_profile block is bigger than
@@ -423,7 +428,9 @@ def test_ff00_jpeg_header(self):
         with Image.open(filename):
             pass
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_truncated_jpeg_should_read_all_the_data(self):
         filename = "Tests/images/truncated_jpeg.jpg"
         ImageFile.LOAD_TRUNCATED_IMAGES = True
@@ -442,7 +449,9 @@ def test_truncated_jpeg_throws_oserror(self):
             with pytest.raises(OSError):
                 im.load()
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_qtables(self, tmp_path):
         def _n_qtables_helper(n, test_file):
             with Image.open(test_file) as im:
@@ -726,7 +735,9 @@ def test_invalid_exif(self):
             # OSError for unidentified image.
             assert im.info.get("dpi") == (72, 72)
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_exif_x_resolution(self, tmp_path):
         with Image.open("Tests/images/flower.jpg") as im:
             exif = im.getexif()
@@ -757,7 +768,9 @@ def test_ifd_offset_exif(self):
             # Act / Assert
             assert im._getexif()[306] == "2017:03:13 23:03:09"
 
-    @pytest.mark.valgrind_known_error(reason="Backtrace in Python Core")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_photoshop(self):
         with Image.open("Tests/images/photoshop-200dpi.jpg") as im:
             assert im.info["photoshop"][0x03ED] == {

Tests/test_file_libtiff.py

@@ -17,6 +17,7 @@
     assert_image_similar,
     assert_image_similar_tofile,
     hopper,
+    mark_if_feature_version,
     skip_unless_feature,
 )
 
@@ -822,13 +823,17 @@ def test_strip_cmyk_16l_jpeg(self):
         with Image.open(infile) as im:
             assert_image_similar_tofile(im, "Tests/images/pil_sample_cmyk.jpg", 0.5)
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_strip_ycbcr_jpeg_2x2_sampling(self):
         infile = "Tests/images/tiff_strip_ycbcr_jpeg_2x2_sampling.tif"
         with Image.open(infile) as im:
             assert_image_similar_tofile(im, "Tests/images/flower.jpg", 0.5)
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_strip_ycbcr_jpeg_1x1_sampling(self):
         infile = "Tests/images/tiff_strip_ycbcr_jpeg_1x1_sampling.tif"
         with Image.open(infile) as im:
@@ -839,13 +844,17 @@ def test_tiled_cmyk_jpeg(self):
         with Image.open(infile) as im:
             assert_image_similar_tofile(im, "Tests/images/pil_sample_cmyk.jpg", 0.5)
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_tiled_ycbcr_jpeg_1x1_sampling(self):
         infile = "Tests/images/tiff_tiled_ycbcr_jpeg_1x1_sampling.tif"
         with Image.open(infile) as im:
             assert_image_equal_tofile(im, "Tests/images/flower2.jpg")
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_tiled_ycbcr_jpeg_2x2_sampling(self):
         infile = "Tests/images/tiff_tiled_ycbcr_jpeg_2x2_sampling.tif"
         with Image.open(infile) as im:

Tests/test_file_pdf.py

@@ -8,7 +8,7 @@
 
 from PIL import Image, PdfParser
 
-from .helper import hopper
+from .helper import hopper, mark_if_feature_version
 
 
 def helper_save_as_pdf(tmp_path, mode, **kwargs):
@@ -85,7 +85,9 @@ def test_unsupported_mode(tmp_path):
         im.save(outfile)
 
 
-@pytest.mark.valgrind_known_error(reason="Known Failing")
+@mark_if_feature_version(
+    pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+)
 def test_save_all(tmp_path):
     # Single frame image
     helper_save_as_pdf(tmp_path, "RGB", save_all=True)

Tests/test_file_png.py

@@ -13,6 +13,7 @@
     hopper,
     is_big_endian,
     is_win32,
+    mark_if_feature_version,
     skip_unless_feature,
 )
 
@@ -679,7 +680,9 @@ def test_exif_save(self, tmp_path):
             exif = reloaded._getexif()
         assert exif[274] == 1
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_exif_from_jpg(self, tmp_path):
         with Image.open("Tests/images/pil_sample_rgb.jpg") as im:
             test_file = str(tmp_path / "temp.png")

Tests/test_file_webp_metadata.py

@@ -4,7 +4,7 @@
 
 from PIL import Image
 
-from .helper import skip_unless_feature
+from .helper import mark_if_feature_version, skip_unless_feature
 
 pytestmark = [
     skip_unless_feature("webp"),
@@ -41,7 +41,9 @@ def test_read_exif_metadata_without_prefix():
         assert exif[305] == "Adobe Photoshop CS6 (Macintosh)"
 
 
-@pytest.mark.valgrind_known_error(reason="Known Failing")
+@mark_if_feature_version(
+    pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+)
 def test_write_exif_metadata():
     file_path = "Tests/images/flower.jpg"
     test_buffer = BytesIO()
@@ -74,7 +76,9 @@ def test_read_icc_profile():
             assert icc == expected_icc
 
 
-@pytest.mark.valgrind_known_error(reason="Known Failing")
+@mark_if_feature_version(
+    pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+)
 def test_write_icc_metadata():
     file_path = "Tests/images/flower2.jpg"
     test_buffer = BytesIO()
@@ -92,7 +96,9 @@ def test_write_icc_metadata():
         assert webp_icc_profile == expected_icc_profile, "Webp ICC didn't match"
 
 
-@pytest.mark.valgrind_known_error(reason="Known Failing")
+@mark_if_feature_version(
+    pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+)
 def test_read_no_exif():
     file_path = "Tests/images/flower.jpg"
     test_buffer = BytesIO()

Tests/test_image.py

@@ -16,6 +16,7 @@
     assert_not_all_same,
     hopper,
     is_win32,
+    mark_if_feature_version,
     skip_unless_feature,
 )
 
@@ -662,7 +663,9 @@ def act(fp):
 
             assert not fp.closed
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_exif_jpeg(self, tmp_path):
         with Image.open("Tests/images/exif-72dpi-int.jpg") as im:  # Little endian
             exif = im.getexif()

Tests/test_image_resample.py

@@ -4,7 +4,12 @@
 
 from PIL import Image, ImageDraw
 
-from .helper import assert_image_equal, assert_image_similar, hopper
+from .helper import (
+    assert_image_equal,
+    assert_image_similar,
+    hopper,
+    mark_if_feature_version,
+)
 
 
 class TestImagingResampleVulnerability:
@@ -455,7 +460,9 @@ def split_range(size, tiles):
                 tiled.paste(tile, (x0, y0))
         return tiled
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_tiles(self):
         with Image.open("Tests/images/flower.jpg") as im:
             assert im.size == (480, 360)
@@ -466,7 +473,9 @@ def test_tiles(self):
                 tiled = self.resize_tiled(im, dst_size, *tiles)
                 assert_image_similar(reference, tiled, 0.01)
 
-    @pytest.mark.valgrind_known_error(reason="Known Failing")
+    @mark_if_feature_version(
+        pytest.mark.valgrind_known_error, "libjpeg_turbo", "2.0", reason="Known Failing"
+    )
     def test_subsample(self):
         # This test shows advantages of the subpixel resizing
         # after supersampling (e.g. during JPEG decoding).

Tests/test_image_thumbnail.py

@@ -88,6 +88,7 @@ def test_no_resize():
         assert im.size == (64, 64)
 
 
+# valgrind test is failing with memory allocated in libjpeg
 @pytest.mark.valgrind_known_error(reason="Known Failing")
 def test_DCT_scaling_edges():
     # Make an image with red borders and size (N * 8) + 1 to cross DCT grid

src/decode.c

@@ -199,7 +199,7 @@ _setimage(ImagingDecoderObject *decoder, PyObject *args) {
             state->bytes = (state->bits * state->xsize + 7) / 8;
         }
         /* malloc check ok, overflow checked above */
-        state->buffer = (UINT8 *)malloc(state->bytes);
+        state->buffer = (UINT8 *)calloc(1, state->bytes);
         if (!state->buffer) {
             return ImagingError_MemoryError();
         }

src/encode.c

@@ -264,7 +264,7 @@ _setimage(ImagingEncoderObject *encoder, PyObject *args) {
         }
         state->bytes = (state->bits * state->xsize + 7) / 8;
         /* malloc check ok, overflow checked above */
-        state->buffer = (UINT8 *)malloc(state->bytes);
+        state->buffer = (UINT8 *)calloc(1, state->bytes);
         if (!state->buffer) {
             return ImagingError_MemoryError();
         }

src/libImaging/Jpeg2KDecode.c

@@ -861,6 +861,10 @@ j2k_decode_entry(Imaging im, ImagingCodecState state) {
                 state->state = J2K_STATE_FAILED;
                 goto quick_exit;
             }
+            /* Undefined behavior, sometimes decode_tile_data doesn't
+               fill the buffer and we do things with it later, leading
+               to valgrind errors. */
+            memset(new, 0, tile_info.data_size);
             state->buffer = new;
             buffer_size = tile_info.data_size;
         }