From 35ed932e5bbfc2b2437c276702bf82b204e44110 Mon Sep 17 00:00:00 2001 From: Vladimir Kempik Date: Mon, 4 Jul 2022 18:34:38 +0300 Subject: [PATCH 1/8] 8289697: buffer overflow in MTLVertexCache.m: MTLVertexCache_AddGlyphQuad --- .../macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m index 35bd357d62d5e..bd080504cdffb 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m +++ b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m @@ -305,7 +305,8 @@ void MTLVertexCache_FreeVertexCache() { J2dTraceLn(J2D_TRACE_INFO, "MTLVertexCache_AddGlyphQuad"); - if (vertexCacheIndex >= MTLVC_MAX_INDEX) + // MTLVC_ADD_TRIANGLES adds 6 vertexes into Cache, so need to check space for 6 elements + if ((vertexCacheIndex + 6) >= MTLVC_MAX_INDEX) { J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex); MTLVertexCache_FlushGlyphVertexCache(); From 88402dbeb709394e602dacee5af4ff4b1a1e8da5 Mon Sep 17 00:00:00 2001 From: Vladimir Kempik Date: Tue, 5 Jul 2022 14:24:29 +0300 Subject: [PATCH 2/8] Add the check to one more place --- .../macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m index bd080504cdffb..a4d80de7af807 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m +++ b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m @@ -219,7 +219,9 @@ void MTLVertexCache_FreeVertexCache() J2dTraceLn1(J2D_TRACE_INFO, "MTLVertexCache_AddMaskQuad: %d", maskCacheIndex); - if (maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) + // MTLVC_ADD_TRIANGLES at the end of this function will place 6 vertexes to the vertex cache + // check free space and flush if needed. + if ((maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) || ((vertexCacheIndex + 6) >= MTLVC_MAX_INDEX)) { J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex); MTLVertexCache_FlushVertexCache(mtlc); From 23438f2747afbf146ac005320d08b7e6e367c42d Mon Sep 17 00:00:00 2001 From: Vladimir Kempik Date: Wed, 6 Jul 2022 00:21:56 +0300 Subject: [PATCH 3/8] Replace constant with define --- .../native/libawt_lwawt/java2d/metal/MTLVertexCache.m | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m index a4d80de7af807..0cb8c44e6d76a 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m +++ b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m @@ -65,6 +65,9 @@ MTLVC_ADD_VERTEX(TX1, TY1, DX1, DY1, 0); \ } while (0) +//Next define should exactly match to the amount of MTLVC_ADD_VERTEX in MTLVC_ADD_TRIANGLES +#define MTL_TRIS_IN_VERTEX 6 + jboolean MTLVertexCache_InitVertexCache() { @@ -219,9 +222,9 @@ void MTLVertexCache_FreeVertexCache() J2dTraceLn1(J2D_TRACE_INFO, "MTLVertexCache_AddMaskQuad: %d", maskCacheIndex); - // MTLVC_ADD_TRIANGLES at the end of this function will place 6 vertexes to the vertex cache + // MTLVC_ADD_TRIANGLES at the end of this function will place MTL_TRIS_IN_VERTEX vertexes to the vertex cache // check free space and flush if needed. - if ((maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) || ((vertexCacheIndex + 6) >= MTLVC_MAX_INDEX)) + if ((maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) || ((vertexCacheIndex + MTL_TRIS_IN_VERTEX) >= MTLVC_MAX_INDEX)) { J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex); MTLVertexCache_FlushVertexCache(mtlc); @@ -307,8 +310,8 @@ void MTLVertexCache_FreeVertexCache() { J2dTraceLn(J2D_TRACE_INFO, "MTLVertexCache_AddGlyphQuad"); - // MTLVC_ADD_TRIANGLES adds 6 vertexes into Cache, so need to check space for 6 elements - if ((vertexCacheIndex + 6) >= MTLVC_MAX_INDEX) + // MTLVC_ADD_TRIANGLES adds MTL_TRIS_IN_VERTEX vertexes into Cache, so need to check space for MTL_TRIS_IN_VERTEX elements + if ((vertexCacheIndex + MTL_TRIS_IN_VERTEX) >= MTLVC_MAX_INDEX) { J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex); MTLVertexCache_FlushGlyphVertexCache(); From 0af4ec814464a29da089b666fb70715837192dc7 Mon Sep 17 00:00:00 2001 From: Vladimir Kempik Date: Wed, 6 Jul 2022 23:26:46 +0300 Subject: [PATCH 4/8] Change define name --- .../native/libawt_lwawt/java2d/metal/MTLVertexCache.m | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m index 0cb8c44e6d76a..28e52881cad59 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m +++ b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m @@ -66,7 +66,7 @@ } while (0) //Next define should exactly match to the amount of MTLVC_ADD_VERTEX in MTLVC_ADD_TRIANGLES -#define MTL_TRIS_IN_VERTEX 6 +#define VERTS_FOR_A_QUAD 6 jboolean MTLVertexCache_InitVertexCache() @@ -224,7 +224,7 @@ void MTLVertexCache_FreeVertexCache() // MTLVC_ADD_TRIANGLES at the end of this function will place MTL_TRIS_IN_VERTEX vertexes to the vertex cache // check free space and flush if needed. - if ((maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) || ((vertexCacheIndex + MTL_TRIS_IN_VERTEX) >= MTLVC_MAX_INDEX)) + if ((maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) || ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX)) { J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex); MTLVertexCache_FlushVertexCache(mtlc); @@ -311,7 +311,7 @@ void MTLVertexCache_FreeVertexCache() J2dTraceLn(J2D_TRACE_INFO, "MTLVertexCache_AddGlyphQuad"); // MTLVC_ADD_TRIANGLES adds MTL_TRIS_IN_VERTEX vertexes into Cache, so need to check space for MTL_TRIS_IN_VERTEX elements - if ((vertexCacheIndex + MTL_TRIS_IN_VERTEX) >= MTLVC_MAX_INDEX) + if ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX) { J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex); MTLVertexCache_FlushGlyphVertexCache(); From 662a6bef7082a0310694e118872b857732dd2894 Mon Sep 17 00:00:00 2001 From: Vladimir Kempik Date: Thu, 7 Jul 2022 18:19:11 +0300 Subject: [PATCH 5/8] Rename comments --- .../macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m index 28e52881cad59..deaac5870a344 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m +++ b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m @@ -222,7 +222,7 @@ void MTLVertexCache_FreeVertexCache() J2dTraceLn1(J2D_TRACE_INFO, "MTLVertexCache_AddMaskQuad: %d", maskCacheIndex); - // MTLVC_ADD_TRIANGLES at the end of this function will place MTL_TRIS_IN_VERTEX vertexes to the vertex cache + // MTLVC_ADD_TRIANGLES at the end of this function will place VERTS_FOR_A_QUAD vertexes to the vertex cache // check free space and flush if needed. if ((maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) || ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX)) { @@ -310,7 +310,7 @@ void MTLVertexCache_FreeVertexCache() { J2dTraceLn(J2D_TRACE_INFO, "MTLVertexCache_AddGlyphQuad"); - // MTLVC_ADD_TRIANGLES adds MTL_TRIS_IN_VERTEX vertexes into Cache, so need to check space for MTL_TRIS_IN_VERTEX elements + // MTLVC_ADD_TRIANGLES adds VERTS_FOR_A_QUAD vertexes into Cache, so need to check space for MTL_TRIS_IN_VERTEX elements if ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX) { J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex); From 2676028f80505e53a5a4463c388b2465a4a5eeee Mon Sep 17 00:00:00 2001 From: Vladimir Kempik Date: Fri, 8 Jul 2022 00:21:28 +0300 Subject: [PATCH 6/8] oops --- .../macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m index deaac5870a344..61e057f847a6b 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m +++ b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m @@ -310,7 +310,7 @@ void MTLVertexCache_FreeVertexCache() { J2dTraceLn(J2D_TRACE_INFO, "MTLVertexCache_AddGlyphQuad"); - // MTLVC_ADD_TRIANGLES adds VERTS_FOR_A_QUAD vertexes into Cache, so need to check space for MTL_TRIS_IN_VERTEX elements + // MTLVC_ADD_TRIANGLES adds VERTS_FOR_A_QUAD vertexes into Cache, so need to check space for VERTS_FOR_A_QUAD elements if ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX) { J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex); From 97a4db25777f4b141f567bfda0532573b6976d10 Mon Sep 17 00:00:00 2001 From: Vladimir Kempik Date: Fri, 8 Jul 2022 10:37:41 +0300 Subject: [PATCH 7/8] break up comments --- .../native/libawt_lwawt/java2d/metal/MTLVertexCache.m | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m index 61e057f847a6b..a56b25d0436b5 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m +++ b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m @@ -65,7 +65,8 @@ MTLVC_ADD_VERTEX(TX1, TY1, DX1, DY1, 0); \ } while (0) -//Next define should exactly match to the amount of MTLVC_ADD_VERTEX in MTLVC_ADD_TRIANGLES +// Next define should exactly match to the amount +// of MTLVC_ADD_VERTEX in MTLVC_ADD_TRIANGLES #define VERTS_FOR_A_QUAD 6 jboolean @@ -222,7 +223,8 @@ void MTLVertexCache_FreeVertexCache() J2dTraceLn1(J2D_TRACE_INFO, "MTLVertexCache_AddMaskQuad: %d", maskCacheIndex); - // MTLVC_ADD_TRIANGLES at the end of this function will place VERTS_FOR_A_QUAD vertexes to the vertex cache + // MTLVC_ADD_TRIANGLES at the end of this function + // will place VERTS_FOR_A_QUAD vertexes to the vertex cache // check free space and flush if needed. if ((maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) || ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX)) { @@ -310,7 +312,8 @@ void MTLVertexCache_FreeVertexCache() { J2dTraceLn(J2D_TRACE_INFO, "MTLVertexCache_AddGlyphQuad"); - // MTLVC_ADD_TRIANGLES adds VERTS_FOR_A_QUAD vertexes into Cache, so need to check space for VERTS_FOR_A_QUAD elements + // MTLVC_ADD_TRIANGLES adds VERTS_FOR_A_QUAD vertexes into Cache + // so need to check space for VERTS_FOR_A_QUAD elements if ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX) { J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex); From ecfbd56380f27146a4ffb6c2808720920a99f0c1 Mon Sep 17 00:00:00 2001 From: Vladimir Kempik Date: Fri, 8 Jul 2022 10:41:19 +0300 Subject: [PATCH 8/8] break up long if --- .../macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m index a56b25d0436b5..fa702ba771273 100644 --- a/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m +++ b/src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m @@ -226,7 +226,8 @@ void MTLVertexCache_FreeVertexCache() // MTLVC_ADD_TRIANGLES at the end of this function // will place VERTS_FOR_A_QUAD vertexes to the vertex cache // check free space and flush if needed. - if ((maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) || ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX)) + if ((maskCacheIndex >= MTLVC_MASK_CACHE_MAX_INDEX) || + ((vertexCacheIndex + VERTS_FOR_A_QUAD) >= MTLVC_MAX_INDEX)) { J2dTraceLn2(J2D_TRACE_INFO, "maskCacheIndex = %d, vertexCacheIndex = %d", maskCacheIndex, vertexCacheIndex); MTLVertexCache_FlushVertexCache(mtlc);