Fix rendering issues when built against the 10.14 SDK.

Before this commit, the Core Text renderer relied on a legacy behavior
of NSView to keep its content across draws. setNeedsDisplayInRect: could
be used to draw over parts of the existing content as drawing commands
were received without needing to redraw old content.

Layer-backed views lose this behavior and may be asked to redraw any or
all of their content at any time, and layer backing becomes the default
for apps built against the macOS 10.14 SDK.

This change adds a way to draw to an intermediate NSImageRep and then
draw that to the view. It's similar to the CGLayer path, but I wasn't
able to get the CGLayer path to work without hanging or crashing when I
scrolled. My best guess from looking at stack traces is that using
CGContextDrawLayerInRect to draw a layer into itself doesn't actually
copy pixels, but adds the self-draw as an action to be performed when
the CGLayer is drawn into a bitmap context. Scrolling stacks these
actions, which either hang or overflow the stack when drawn.

The new code is controlled by the MMDrawToImage user default, which is
on by default in this change. Fixes macvim-dev#751.

src/MacVim/MMAppController.m

@@ -234,6 +234,7 @@ + (void)initialize
         [NSNumber numberWithBool:YES],    MMNativeFullScreenKey,
         [NSNumber numberWithDouble:0.25], MMFullScreenFadeTimeKey,
         [NSNumber numberWithBool:NO],     MMUseCGLayerAlwaysKey,
+        [NSNumber numberWithBool:YES],    MMDrawToImageKey,
         [NSNumber numberWithBool:YES],    MMShareFindPboardKey,
         nil];
 

src/MacVim/MMCoreTextView.h

@@ -46,6 +46,8 @@
     CGContextRef                cgLayerContext;
     NSLock                      *cgLayerLock;
 
+    NSBitmapImageRep            *contentsImageRep;
+
     // These are used in MMCoreTextView+ToolTip.m
     id trackingRectOwner_;              // (not retained)
     void *trackingRectUserData_;

src/MacVim/MMCoreTextView.m

@@ -168,6 +168,8 @@ - (void)dealloc
     [drawData release];  drawData = nil;
     [fontCache release];  fontCache = nil;
 
+    [contentsImageRep release];  contentsImageRep = nil;
+
     [helper setTextView:nil];
     [helper release];  helper = nil;
 
@@ -444,11 +446,18 @@ - (BOOL)_wantsKeyDownForEvent:(id)event
 }
 
 - (void)setFrameSize:(NSSize)newSize {
-    if (!drawPending && !NSEqualSizes(newSize, self.frame.size) && drawData.count == 0) {
+    if (NSEqualSizes(newSize, self.bounds.size))
+        return;
+    if (!drawPending) {
         [NSAnimationContext beginGrouping];
         drawPending = YES;
     }
     [super setFrameSize:newSize];
+    if (contentsImageRep || [[NSUserDefaults standardUserDefaults]
+                             boolForKey:MMDrawToImageKey]) {
+        [contentsImageRep release];
+        contentsImageRep = [[self bitmapImageRepForCachingDisplayInRect:self.bounds] retain];
+    }
 }
 
 - (void)keyDown:(NSEvent *)event
@@ -611,7 +620,19 @@ - (void)drawRect:(NSRect)rect
     NSGraphicsContext *context = [NSGraphicsContext currentContext];
     [context setShouldAntialias:antialias];
 
-    if (cgLayerEnabled && drawData.count == 0) {
+    if (contentsImageRep) {
+        const NSRect *rects;
+        NSInteger count;
+        [self getRectsBeingDrawn:&rects count:&count];
+        for (size_t i = 0; i < count; i++) {
+            [contentsImageRep drawInRect:rects[i]
+                                fromRect:rects[i]
+                               operation:NSCompositingOperationCopy
+                                fraction:1.0
+                          respectFlipped:YES
+                                   hints:nil];
+        }
+    } else if (cgLayerEnabled && drawData.count == 0) {
         // during a live resize, we will have around a stale layer until the
         // refresh messages travel back from the vim process. We push the old
         // layer in at an offset to get rid of jitter due to lines changing
@@ -653,7 +674,12 @@ - (void)drawRect:(NSRect)rect
 
 - (void)performBatchDrawWithData:(NSData *)data
 {
-    if (cgLayerEnabled && drawData.count == 0 && [self getCGContext]) {
+    if (contentsImageRep) {
+        [NSGraphicsContext saveGraphicsState];
+        NSGraphicsContext.currentContext = [NSGraphicsContext graphicsContextWithBitmapImageRep:contentsImageRep];
+        [self batchDrawData:data];
+        [NSGraphicsContext restoreGraphicsState];
+    } else if (cgLayerEnabled && drawData.count == 0 && [self getCGContext]) {
         [cgLayerLock lock];
         [self batchDrawData:data];
         [cgLayerLock unlock];
@@ -662,13 +688,20 @@ - (void)performBatchDrawWithData:(NSData *)data
         [self setNeedsDisplay:YES];
     }
     if (drawPending) {
-        [NSAnimationContext endGrouping];
-        drawPending = NO;
+        int r = maxRows, c = maxColumns;
+        [self constrainRows:&r columns:&c toSize:[self desiredSize]];
+        if (r == maxRows && c == maxColumns) {
+            [NSAnimationContext endGrouping];
+            drawPending = NO;
+        }
     }
 }
 
 - (void)setCGLayerEnabled:(BOOL)enabled
 {
+    if (contentsImageRep)
+        return;
+
     cgLayerEnabled = enabled;
 
     if (!cgLayerEnabled)
@@ -710,13 +743,13 @@ - (CGContextRef)getCGContext
 
 - (void)setNeedsDisplayCGLayerInRect:(CGRect)rect
 {
-    if (cgLayerEnabled)
+    if (cgLayerEnabled || contentsImageRep)
        [self setNeedsDisplayInRect:rect];
 }
 
 - (void)setNeedsDisplayCGLayer:(BOOL)flag
 {
-    if (cgLayerEnabled)
+    if (cgLayerEnabled || contentsImageRep)
        [self setNeedsDisplay:flag];
 }
 
@@ -1491,7 +1524,16 @@ - (void)drawString:(const UniChar *)chars length:(UniCharCount)length
 
 - (void)scrollRect:(NSRect)rect lineCount:(int)count
 {
-    if (cgLayerEnabled) {
+    if (contentsImageRep) {
+        NSRect toRect = NSOffsetRect(rect, 0, -count * cellSize.height);
+        [contentsImageRep drawInRect:toRect
+                            fromRect:rect
+                           operation:NSCompositingOperationCopy
+                            fraction:1.0
+                      respectFlipped:NO
+                               hints:nil];
+        [self setNeedsDisplayCGLayerInRect:toRect];
+    } else if (cgLayerEnabled) {
         CGContextRef context = [self getCGContext];
         int yOffset = count * cellSize.height;
         NSRect clipRect = rect;

src/MacVim/Miscellaneous.h

@@ -53,6 +53,7 @@ extern NSString *MMNativeFullScreenKey;
 extern NSString *MMUseMouseTimeKey;
 extern NSString *MMFullScreenFadeTimeKey;
 extern NSString *MMUseCGLayerAlwaysKey;
+extern NSString *MMDrawToImageKey;
 
 
 // Enum for MMUntitledWindowKey

src/MacVim/Miscellaneous.m

@@ -49,6 +49,7 @@
 NSString *MMUseMouseTimeKey             = @"MMUseMouseTime";
 NSString *MMFullScreenFadeTimeKey       = @"MMFullScreenFadeTime";
 NSString *MMUseCGLayerAlwaysKey         = @"MMUseCGLayerAlways";
+NSString *MMDrawToImageKey               = @"MMDrawToImage";