8348989: Better Glyph drawing

Reviewed-by: avoitylov, mbalao, andrew
Backport-of: 191c37a280faccfaecae033a68313ad06cdfc411

Author: Ilarion Nakonechnyy

src/java.desktop/macosx/native/libawt_lwawt/font/CGGlyphImages.m

@@ -237,6 +237,7 @@ @implementation CGGI_GlyphCanvas
 {
     UInt32 *src = (UInt32 *)canvas->image->data;
     size_t srcRowWidth = canvas->image->width;
+    size_t srcHeight = canvas->image->height;
 
     UInt8 *dest = (UInt8 *)info->image;
     size_t destRowWidth = info->width;
@@ -246,12 +247,12 @@ @implementation CGGI_GlyphCanvas
     size_t y;
 
     // fill empty glyph image with black-on-white glyph
-    for (y = 0; y < height; y++) {
+    for (y = 0; y < height && y < srcHeight; y++) {
         size_t destRow = y * destRowWidth * 3;
         size_t srcRow = y * srcRowWidth;
 
         size_t x;
-        for (x = 0; x < destRowWidth; x++) {
+        for (x = 0; x < destRowWidth && x < srcRowWidth; x++) {
             CGGI_CopyARGBPixelToRGBPixel(src[srcRow + x],
                                          dest + destRow + x * 3);
         }
@@ -289,6 +290,7 @@ @implementation CGGI_GlyphCanvas
 {
     UInt32 *src = (UInt32 *)canvas->image->data;
     size_t srcRowWidth = canvas->image->width;
+    size_t srcHeight = canvas->image->height;
 
     UInt8 *dest = (UInt8 *)info->image;
     size_t destRowWidth = info->width;
@@ -298,11 +300,11 @@ @implementation CGGI_GlyphCanvas
     size_t y;
 
     // fill empty glyph image with black-on-white glyph
-    for (y = 0; y < height; y++) {
+    for (y = 0; y < height && y < srcHeight; y++) {
         size_t destRow = y * destRowWidth;
         size_t srcRow = y * srcRowWidth;
         size_t x;
-        for (x = 0; x < destRowWidth; x++) {
+        for (x = 0; x < destRowWidth && x < srcRowWidth; x++) {
             UInt32 p = src[srcRow + x];
             dest[destRow + x] = CGGI_ConvertBWPixelToByteGray(p);
         }
@@ -317,6 +319,7 @@ @implementation CGGI_GlyphCanvas
 
     UInt32 *src = (UInt32 *)canvas->image->data;
     size_t srcRowWidth = canvas->image->width;
+    size_t srcHeight = canvas->image->height;
 
     UInt8 *dest = (UInt8 *)info->image;
     size_t destRowWidth = info->width;
@@ -325,15 +328,16 @@ @implementation CGGI_GlyphCanvas
 
     size_t y;
 
-    for (y = 0; y < height; y++) {
+    for (y = 0; y < height && y < srcHeight; y++) {
         size_t srcRow = y * srcRowWidth;
         if (littleEndian) {
-            UInt16 destRowBytes = info->rowBytes;
+            size_t srcRowBytes = canvas->image->rowBytes;
+            UInt16 destRowBytes = (info->rowBytes < srcRowBytes) ? info->rowBytes : srcRowBytes;
             memcpy(dest, src + srcRow, destRowBytes);
             dest += destRowBytes;
         } else {
             size_t x;
-            for (x = 0; x < destRowWidth; x++) {
+            for (x = 0; x < destRowWidth && x < srcRowWidth; x++) {
                 UInt32 p = src[srcRow + x];
                 *dest++ = (p >> 24  & 0xFF); // blue  (alpha-premultiplied)
                 *dest++ = (p >> 16 & 0xFF); // green (alpha-premultiplied)
@@ -426,8 +430,10 @@ @implementation CGGI_GlyphCanvas
 
     canvas->image->data = (void *)calloc(byteCount, sizeof(UInt8));
     if (canvas->image->data == NULL) {
-        [[NSException exceptionWithName:NSMallocException
-            reason:@"Failed to allocate memory for the buffer which backs the CGContext for glyph strikes." userInfo:nil] raise];
+        canvas->image->width = 0;
+        canvas->image->height = 0;
+        canvas->image->rowBytes = 0;
+        canvas->image->data = malloc(0);
     }
 
     uint32_t bmpInfo = kCGImageAlphaPremultipliedFirst;
@@ -477,25 +483,42 @@ @implementation CGGI_GlyphCanvas
 
 /*
  * Quick and easy inline to check if this canvas is big enough.
+ * This function only increases the size. To get a smaller canvas, free it first.
+ * This function adds padding / slack multiplier to the requested size.
+ * So resizes must be based on the size you need, not the size of the canvas.
+ * The function will internally account for the multiplier it uses.
  */
 static inline void
 CGGI_SizeCanvas(CGGI_GlyphCanvas *canvas, const vImagePixelCount width,
         const vImagePixelCount height,
         const CGGI_RenderingMode* mode)
 {
     if (canvas->image != NULL &&
-        width  < canvas->image->width &&
-        height < canvas->image->height)
+        width  * CGGI_GLYPH_CANVAS_SLACK <= canvas->image->width &&
+        height * CGGI_GLYPH_CANVAS_SLACK <= canvas->image->height)
     {
         return;
     }
 
+    vImagePixelCount w = width * CGGI_GLYPH_CANVAS_SLACK;
+    vImagePixelCount h = height * CGGI_GLYPH_CANVAS_SLACK;
+
+    // Do not allow the canvas to be resized smaller.
+    if (canvas->image != NULL) {
+       if (w < canvas->image->width) {
+           w = canvas->image->width;
+       }
+       if (h < canvas->image->height) {
+           h = canvas->image->height;
+       }
+    }
+
     // if we don't have enough space to strike the largest glyph in the
     // run, resize the canvas
     CGGI_FreeCanvas(canvas);
     CGGI_InitCanvas(canvas,
-                    width * CGGI_GLYPH_CANVAS_SLACK,
-                    height * CGGI_GLYPH_CANVAS_SLACK,
+                    w,
+                    h,
                     mode);
     JRSFontSetRenderingStyleOnContext(canvas->context, mode->cgFontMode);
 }
@@ -511,6 +534,12 @@ @implementation CGGI_GlyphCanvas
     canvasRectToClear.data = canvas->image->data;
     canvasRectToClear.height = info->height;
     canvasRectToClear.width = info->width;
+    if (canvas->image->width < canvasRectToClear.width) {
+          canvasRectToClear.width = canvas->image->width;
+    }
+    if (canvas->image->height < canvasRectToClear.height) {
+          canvasRectToClear.height = canvas->image->height;
+    }
     // use the row stride of the canvas, not the info
     canvasRectToClear.rowBytes = canvas->image->rowBytes;
 
@@ -870,7 +899,6 @@ @implementation CGGI_GlyphCanvas
         CGRect bbox = bboxes[i];
 
         GlyphInfo *glyphInfo = CGGI_CreateNewGlyphInfoFrom(advance, bbox, strike, mainFontDescriptor);
-
         if (maxWidth < glyphInfo->width)   maxWidth = glyphInfo->width;
         if (maxHeight < glyphInfo->height) maxHeight = glyphInfo->height;
 
@@ -947,8 +975,7 @@ @implementation CGGI_GlyphCanvas
     }
 
     // just do one malloc, and carve it up for all the buffers
-    void *buffer = malloc(sizeof(CGRect) * sizeof(CGSize) *
-                          sizeof(CGGlyph) * sizeof(UnicodeScalarValue) * len);
+    void *buffer = malloc((sizeof(CGRect) + sizeof(CGSize) + sizeof(CGGlyph) + sizeof(UnicodeScalarValue)) * len);
     if (buffer == NULL) {
         [[NSException exceptionWithName:NSMallocException
             reason:@"Failed to allocate memory for the temporary glyph strike and measurement buffers." userInfo:nil] raise];
@@ -966,6 +993,8 @@ @implementation CGGI_GlyphCanvas
     free(buffer);
 }
 
+#define TX_FIXED_UNSAFE(v)  (isinf(v) || isnan(v) || fabs(v) >= (1<<30))
+
 /*
  * Calculates bounding boxes (for given transform) and advance (for untransformed 1pt-size font) for specified glyphs.
  */
@@ -977,6 +1006,27 @@ @implementation CGGI_GlyphCanvas
                               size_t count,
                               CGRect bboxes[],
                               CGSize advances[]) {
+
+    if (TX_FIXED_UNSAFE(tx->a) || TX_FIXED_UNSAFE(tx->b) || TX_FIXED_UNSAFE(tx->c) ||
+        TX_FIXED_UNSAFE(tx->d) || TX_FIXED_UNSAFE(tx->tx) || TX_FIXED_UNSAFE(tx->tx)) {
+
+        if (bboxes) {
+            for (int i = 0; i < count; i++) {
+                bboxes[i].origin.x = 0;
+                bboxes[i].origin.y = 0;
+                bboxes[i].size.width = 0;
+                bboxes[i].size.height = 0;
+            }
+        }
+        if (advances) {
+            for (int i = 0; i < count; i++) {
+                advances[i].width = 0;
+                advances[i].height = 0;
+            }
+        }
+        return;
+    }
+
     if (IsEmojiFont(font)) {
         // Glyph metrics for emoji font are not strictly proportional to font size,
         // so we need to construct real-sized font object to calculate them.