Fix segfault when scaled image dimension is less than clip's

RPCS3 · Dec 4, 2018 · 45942c4 · 45942c4
1 parent fa5652f
commit 45942c4
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 14 deletions.
diff --git a/rpcs3/Emu/RSX/rsx_methods.cpp b/rpcs3/Emu/RSX/rsx_methods.cpp
@@ -1048,7 +1048,9 @@ namespace rsx
 					{
 						if (need_convert)
 						{
-							convert_scale_image(temp2, out_format, convert_w, convert_h, out_pitch,
+							temp2.reset(new u8[out_pitch * (std::max(convert_h, (u32)clip_h) - 1) + (out_bpp * std::max(convert_w, (u32)clip_w))]);
+
+							convert_scale_image(temp2.get(), out_format, convert_w, convert_h, out_pitch,
 								pixels_src, in_format, in_w, in_h, in_pitch, slice_h, in_inter == blit_engine::transfer_interpolator::foh);
 
 							clip_image(pixels_dst, temp2.get(), clip_x, clip_y, clip_w, clip_h, out_bpp, out_pitch, out_pitch);
@@ -1090,7 +1092,9 @@ namespace rsx
 					{
 						if (need_convert)
 						{
-							convert_scale_image(temp2, out_format, convert_w, convert_h, out_pitch,
+							temp2.reset(new u8[out_pitch * (std::max(convert_h, (u32)clip_h) - 1) + (out_bpp * std::max(convert_w, (u32)clip_w))]);
+
+							convert_scale_image(temp2.get(), out_format, convert_w, convert_h, out_pitch,
 								pixels_src, in_format, in_w, in_h, in_pitch, slice_h, in_inter == blit_engine::transfer_interpolator::foh);
 
 							clip_image(temp3, temp2.get(), clip_x, clip_y, clip_w, clip_h, out_bpp, out_pitch, out_pitch);
@@ -1102,7 +1106,9 @@ namespace rsx
 					}
 					else
 					{
-						convert_scale_image(temp3, out_format, out_w, out_h, out_pitch,
+						temp3.reset(new u8[out_pitch * (out_h - 1) + (out_bpp * out_w)]);
+
+						convert_scale_image(temp3.get(), out_format, out_w, out_h, out_pitch,
 							pixels_src, in_format, in_w, in_h, in_pitch, clip_h, in_inter == blit_engine::transfer_interpolator::foh);
 					}
 

diff --git a/rpcs3/Emu/RSX/rsx_utils.cpp b/rpcs3/Emu/RSX/rsx_utils.cpp
@@ -23,14 +23,6 @@ namespace rsx
 		sws_scale(sws.get(), &src, &src_pitch, 0, src_slice_h, &dst, &dst_pitch);
 	}
 
-	void convert_scale_image(std::unique_ptr<u8[]>& dst, AVPixelFormat dst_format, int dst_width, int dst_height, int dst_pitch,
-		const u8 *src, AVPixelFormat src_format, int src_width, int src_height, int src_pitch, int src_slice_h, bool bilinear)
-	{
-		dst.reset(new u8[dst_pitch * dst_height]);
-		convert_scale_image(dst.get(), dst_format, dst_width, dst_height, dst_pitch,
-			src, src_format, src_width, src_height, src_pitch, src_slice_h, bilinear);
-	}
-
 	void clip_image(u8 *dst, const u8 *src, int clip_x, int clip_y, int clip_w, int clip_h, int bpp, int src_pitch, int dst_pitch)
 	{
 		u8 *pixels_src = (u8*)src + clip_y * src_pitch + clip_x * bpp;

diff --git a/rpcs3/Emu/RSX/rsx_utils.h b/rpcs3/Emu/RSX/rsx_utils.h
@@ -307,9 +307,6 @@ namespace rsx
 	void convert_scale_image(u8 *dst, AVPixelFormat dst_format, int dst_width, int dst_height, int dst_pitch,
 		const u8 *src, AVPixelFormat src_format, int src_width, int src_height, int src_pitch, int src_slice_h, bool bilinear);
 
-	void convert_scale_image(std::unique_ptr<u8[]>& dst, AVPixelFormat dst_format, int dst_width, int dst_height, int dst_pitch,
-		const u8 *src, AVPixelFormat src_format, int src_width, int src_height, int src_pitch, int src_slice_h, bool bilinear);
-
 	void clip_image(u8 *dst, const u8 *src, int clip_x, int clip_y, int clip_w, int clip_h, int bpp, int src_pitch, int dst_pitch);
 	void clip_image(std::unique_ptr<u8[]>& dst, const u8 *src, int clip_x, int clip_y, int clip_w, int clip_h, int bpp, int src_pitch, int dst_pitch);