8252726: Lanai: IDEA Editor Rendering OGL vs Metal 1:2

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLMaskFill.m

@@ -90,8 +90,8 @@
                      unsigned char *pMask)
 {
     J2dTraceLn5(J2D_TRACE_INFO, "MTLMaskFill_MaskFill (x=%d y=%d w=%d h=%d pMask=%p)", x, y, w, h, dstOps->pTexture);
-
-    MTLVertexCache_EnableMaskCache(mtlc, dstOps);
+    if (mtlc == nil) return;
+    CHECK_PREVIOUS_OP(MTL_OP_MASK_OP);
     jint tw, th, x0;
     jint sx1, sy1, sx2, sy2;
     jint sx, sy, sw, sh;
@@ -116,8 +116,6 @@
                     maskscan, pMask, dstOps);
         }
     }
-
-    MTLVertexCache_DisableMaskCache(mtlc);
 }
 
 JNIEXPORT void JNICALL

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLRenderQueue.h

@@ -50,9 +50,10 @@ enum {
   MTL_OP_RESET_PAINT,
   MTL_OP_SYNC,
   MTL_OP_SHAPE_CLIP_SPANS,
+  MTL_OP_MASK_OP,
   MTL_OP_OTHER
 };
-
+void MTLRenderQueue_CheckPreviousOp(jint op);
 /*
  * These macros now simply delegate to the CheckPreviousOp() method.
  */

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLRenderQueue.m

@@ -315,18 +315,28 @@ void MTLRenderQueue_CheckPreviousOp(jint op) {
         return;
     }
 
-    J2dTraceLn1(J2D_TRACE_VERBOSE,
-                "MTLRenderQueue_CheckPreviousOp: new op=%d", op);
-
     if (op == MTL_OP_SET_COLOR) {
-        return; // SET_COLOR should not cause endEncoder
-    }
-
-    if (mtlPreviousOp == MTL_OP_INIT) {
+        if (mtlPreviousOp != MTL_OP_MASK_OP) {
+            return; // SET_COLOR should not cause endEncoder
+        }
+    } else if (op == MTL_OP_MASK_OP) {
+        MTLVertexCache_EnableMaskCache(mtlc, dstOps);
         mtlPreviousOp = op;
         return;
     }
 
+    J2dTraceLn1(J2D_TRACE_VERBOSE,
+                "MTLRenderQueue_CheckPreviousOp: new op=%d", op);
+
+    switch (mtlPreviousOp) {
+        case MTL_OP_INIT :
+            mtlPreviousOp = op;
+            return;
+        case MTL_OP_MASK_OP :
+            MTLVertexCache_DisableMaskCache(mtlc);
+            break;
+    }
+
     if (mtlc != NULL) {
         [mtlc.encoderManager endEncoder];
 
@@ -711,7 +721,6 @@ void MTLRenderQueue_CheckPreviousOp(jint op) {
                 }
                 case sun_java2d_pipe_BufferedOpCodes_MASK_FILL:
                 {
-                    CHECK_PREVIOUS_OP(MTL_OP_OTHER);
                     jint x        = NEXT_INT(b);
                     jint y        = NEXT_INT(b);
                     jint w        = NEXT_INT(b);
@@ -1137,6 +1146,9 @@ void MTLRenderQueue_CheckPreviousOp(jint op) {
         }
 
         if (mtlc != NULL) {
+            if (mtlPreviousOp == MTL_OP_MASK_OP) {
+                MTLVertexCache_DisableMaskCache(mtlc);
+            }
             [mtlc.encoderManager endEncoder];
             MTLCommandBufferWrapper * cbwrapper = [mtlc pullCommandBufferWrapper];
             id<MTLCommandBuffer> commandbuf = [cbwrapper getCommandBuffer];

src/java.desktop/macosx/native/libawt_lwawt/java2d/metal/MTLVertexCache.m

@@ -239,11 +239,6 @@ void MTLVertexCache_FreeVertexCache()
     J2dTraceLn(J2D_TRACE_INFO, "MTLVertexCache_DisableMaskCache");
     MTLVertexCache_FlushVertexCache(mtlc);
     MTLVertexCache_RestoreColorState(mtlc);
-    if (maskCacheTex != nil) {
-        [[mtlc getCommandBufferWrapper] registerPooledTexture:maskCacheTex];
-        [maskCacheTex release];
-        maskCacheTex = nil;
-    }
     maskCacheIndex = 0;
     free(vertexCache);
     vertexCache = NULL;