8315074: Possible null pointer access in native glass

jayathirthrao · kevinrushforth · commit f7b21e5468f1 · 2023-09-13T13:26:34.000Z
Reviewed-by: jvos, kcr
diff --git a/modules/javafx.graphics/src/main/native-glass/gtk/GlassApplication.cpp b/modules/javafx.graphics/src/main/native-glass/gtk/GlassApplication.cpp
@@ -262,8 +262,13 @@ JNIEXPORT void JNICALL Java_com_sun_glass_ui_gtk_GtkApplication__1submitForLater
     (void)obj;
 
     RunnableContext* context = (RunnableContext*)malloc(sizeof(RunnableContext));
-    context->runnable = env->NewGlobalRef(runnable);
-    gdk_threads_add_idle_full(G_PRIORITY_HIGH_IDLE + 30, call_runnable, context, NULL);
+    if (context != NULL) {
+        context->runnable = env->NewGlobalRef(runnable);
+        gdk_threads_add_idle_full(G_PRIORITY_HIGH_IDLE + 30, call_runnable, context, NULL);
+        // we release this context in call_runnable
+    } else {
+        fprintf(stderr, "malloc failed in GtkApplication__1submitForLaterInvocation\n");
+    }
 }
 
 /*
diff --git a/modules/javafx.graphics/src/main/native-glass/gtk/GlassTimer.cpp b/modules/javafx.graphics/src/main/native-glass/gtk/GlassTimer.cpp
@@ -45,10 +45,16 @@ JNIEXPORT jlong JNICALL Java_com_sun_glass_ui_gtk_GtkTimer__1start
     (void)obj;
 
     RunnableContext* context = (RunnableContext*) malloc(sizeof(RunnableContext));
-    context->runnable = env->NewGlobalRef(runnable);
-    context->flag = 0;
-    gdk_threads_add_timeout_full(G_PRIORITY_HIGH_IDLE, period, call_runnable_in_timer, context, NULL);
-    return PTR_TO_JLONG(context);
+    if (context != NULL) {
+        context->runnable = env->NewGlobalRef(runnable);
+        context->flag = 0;
+        gdk_threads_add_timeout_full(G_PRIORITY_HIGH_IDLE, period, call_runnable_in_timer, context, NULL);
+        return PTR_TO_JLONG(context);
+    } else {
+        // we throw RuntimeException on Java side when we can't
+        // start the timer
+        return 0L;
+    }
 }
 
 /*
diff --git a/modules/javafx.graphics/src/main/native-glass/gtk/glass_general.h b/modules/javafx.graphics/src/main/native-glass/gtk/glass_general.h
@@ -96,6 +96,12 @@ struct jni_exception: public std::exception {
     jstring jmessage;
 };
 
+#define SAFE_FREE(PTR)  \
+    if ((PTR) != NULL) {  \
+        free(PTR);     \
+        (PTR) = NULL;     \
+    }
+
 #define EXCEPTION_OCCURED(env) (check_and_clear_exception(env))
 
 #define CHECK_JNI_EXCEPTION(env) \
diff --git a/modules/javafx.graphics/src/main/native-glass/gtk/glass_window_ime.cpp b/modules/javafx.graphics/src/main/native-glass/gtk/glass_window_ime.cpp
@@ -59,6 +59,12 @@ bool WindowContextBase::im_filter_keypress(GdkEventKey* event) {
         buffer = (char*)malloc(buf_len * sizeof (char));
     }
 
+    if (buffer == NULL) {
+        // dont process the key event
+        fprintf(stderr, "malloc failed in im_filter_keypress\n");
+        return false;
+    }
+
     KeySym keysym;
     Status status;
     XKeyPressedEvent xevent = convert_event(event);
@@ -74,7 +80,14 @@ bool WindowContextBase::im_filter_keypress(GdkEventKey* event) {
     int len = Xutf8LookupString(xim.ic, &xevent, buffer, buf_len - 1, &keysym, &status);
     if (status == XBufferOverflow) {
         buf_len = len + 1;
-        buffer = (char*)realloc(buffer, buf_len * sizeof (char));
+        char *tmpBuffer = (char*)realloc(buffer, buf_len * sizeof (char));
+        if (tmpBuffer == NULL) {
+            SAFE_FREE(buffer);
+            // dont process the key event
+            fprintf(stderr, "realloc failed in im_filter_keypress\n");
+            return false;
+        }
+        buffer = tmpBuffer;
         len = Xutf8LookupString(xim.ic, &xevent, buffer, buf_len - 1,
                 &keysym, &status);
     }
diff --git a/modules/javafx.graphics/src/main/native-glass/ios/GlassMacros.h b/modules/javafx.graphics/src/main/native-glass/ios/GlassMacros.h
@@ -73,22 +73,34 @@ do {
     GlassThreadData *_GlassThreadData = (GlassThreadData*)pthread_getspecific(GlassThreadDataKey); \
     if (_GlassThreadData == NULL) \
     { \
-        _GlassThreadData = malloc(sizeof(GlassThreadData)); \
-        memset(_GlassThreadData, 0x00, sizeof(GlassThreadData)); \
-        pthread_setspecific(GlassThreadDataKey, _GlassThreadData); \
+        _GlassThreadData = calloc(1, sizeof(GlassThreadData)); \
+        if (_GlassThreadData != NULL) \
+        { \
+            pthread_setspecific(GlassThreadDataKey, _GlassThreadData); \
+        } \
+        else \
+        { \
+            fprintf(stderr, "malloc failed in GLASS_POOL_ENTER\n"); \
+        } \
     } \
-    assert(_GlassThreadData->counter >= 0); \
-    if (_GlassThreadData->counter++ == 0) \
+    if (_GlassThreadData != NULL) \
     { \
-        _GlassThreadData->pool = [[NSAutoreleasePool alloc] init]; \
+        assert(_GlassThreadData->counter >= 0); \
+        if (_GlassThreadData->counter++ == 0) \
+        { \
+            _GlassThreadData->pool = [[NSAutoreleasePool alloc] init]; \
+        } \
     }
 #define GLASS_POOL_EXIT \
-    if (--_GlassThreadData->counter == 0) \
+    if (_GlassThreadData != NULL) \
     { \
-        [_GlassThreadData->pool drain]; \
-        _GlassThreadData->pool = nil; \
+        if (--_GlassThreadData->counter == 0) \
+        { \
+            [_GlassThreadData->pool drain]; \
+            _GlassThreadData->pool = nil; \
+        } \
+        assert(_GlassThreadData->counter >= 0); \
     } \
-    assert(_GlassThreadData->counter >= 0); \
 }
 
 // variations of GLASS_POOL_ENTER/GLASS_POOL_EXIT allowing them to be used accross different calls
diff --git a/modules/javafx.graphics/src/main/native-glass/mac/GlassApplication.m b/modules/javafx.graphics/src/main/native-glass/mac/GlassApplication.m
@@ -362,6 +362,7 @@ - (void)application:(NSApplication *)theApplication openFiles:(NSArray *)filenam
     LOG("GlassApplication:application:openFiles");
 
     GET_MAIN_JENV;
+    NSApplicationDelegateReply reply = NSApplicationDelegateReplySuccess;
     NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
     {
         NSUInteger count = [filenames count];
@@ -371,21 +372,26 @@ - (void)application:(NSApplication *)theApplication openFiles:(NSArray *)filenam
         }
         jobjectArray files = (*env)->NewObjectArray(env, (jsize)count, stringClass, NULL);
         GLASS_CHECK_EXCEPTION(env);
-        for (NSUInteger i=0; i<count; i++)
-        {
-            NSString *file = [filenames objectAtIndex:i];
-            if (file != nil)
+        if (files != NULL) {
+            for (NSUInteger i=0; i<count; i++)
             {
-                (*env)->SetObjectArrayElement(env, files, (jsize)i, (*env)->NewStringUTF(env, [file UTF8String]));
-                GLASS_CHECK_EXCEPTION(env);
+                NSString *file = [filenames objectAtIndex:i];
+                if (file != nil)
+                {
+                    (*env)->SetObjectArrayElement(env, files, (jsize)i, (*env)->NewStringUTF(env, [file UTF8String]));
+                    GLASS_CHECK_EXCEPTION(env);
+                }
             }
+            (*env)->CallVoidMethod(env, self->jApplication, [GlassHelper ApplicationNotifyOpenFilesMethod], files);
+        } else {
+            fprintf(stderr, "NewObjectArray failed in GlassApplication_application\n");
+            reply = NSApplicationDelegateReplyFailure;
         }
-        (*env)->CallVoidMethod(env, self->jApplication, [GlassHelper ApplicationNotifyOpenFilesMethod], files);
     }
     [pool drain];
     GLASS_CHECK_EXCEPTION(env);
 
-    [theApplication replyToOpenOrPrint:NSApplicationDelegateReplySuccess];
+    [theApplication replyToOpenOrPrint:reply];
 }
 
 - (BOOL)application:(NSApplication *)theApplication openFile:(NSString *)filename
diff --git a/modules/javafx.graphics/src/main/native-glass/mac/GlassKey.m b/modules/javafx.graphics/src/main/native-glass/mac/GlassKey.m
@@ -461,6 +461,9 @@ jcharArray GetJavaKeyChars(JNIEnv *env, NSEvent *event)
     jchar jc[16];
     [chars getCharacters:jc range:NSMakeRange(0, [chars length])];
     jcharArray jChars = (*env)->NewCharArray(env, (jsize)[chars length]);
+    if (jChars == NULL) {
+        return NULL;
+    }
     (*env)->SetCharArrayRegion(env, jChars, 0, (jsize)[chars length], jc);
     GLASS_CHECK_EXCEPTION(env);
     return jChars;
diff --git a/modules/javafx.graphics/src/main/native-glass/mac/GlassMacros.h b/modules/javafx.graphics/src/main/native-glass/mac/GlassMacros.h
@@ -177,22 +177,34 @@ do {
     GlassThreadData *_GlassThreadData = (GlassThreadData*)pthread_getspecific(GlassThreadDataKey); \
     if (_GlassThreadData == NULL) \
     { \
-        _GlassThreadData = malloc(sizeof(GlassThreadData)); \
-        memset(_GlassThreadData, 0x00, sizeof(GlassThreadData)); \
-        pthread_setspecific(GlassThreadDataKey, _GlassThreadData); \
+        _GlassThreadData = calloc(1, sizeof(GlassThreadData)); \
+        if (_GlassThreadData != NULL) \
+        { \
+            pthread_setspecific(GlassThreadDataKey, _GlassThreadData); \
+        } \
+        else \
+        { \
+            fprintf(stderr, "malloc failed in GLASS_POOL_ENTER\n"); \
+        } \
     } \
-    assert(_GlassThreadData->counter >= 0); \
-    if (_GlassThreadData->counter++ == 0) \
+    if (_GlassThreadData != NULL) \
     { \
-        _GlassThreadData->pool = [[NSAutoreleasePool alloc] init]; \
+        assert(_GlassThreadData->counter >= 0); \
+        if (_GlassThreadData->counter++ == 0) \
+        { \
+            _GlassThreadData->pool = [[NSAutoreleasePool alloc] init]; \
+        } \
     }
 #define GLASS_POOL_EXIT \
-    if (--_GlassThreadData->counter == 0) \
+    if (_GlassThreadData != NULL) \
     { \
-        [_GlassThreadData->pool drain]; \
-        _GlassThreadData->pool = nil; \
+        if (--_GlassThreadData->counter == 0) \
+        { \
+            [_GlassThreadData->pool drain]; \
+            _GlassThreadData->pool = nil; \
+        } \
+        assert(_GlassThreadData->counter >= 0); \
     } \
-    assert(_GlassThreadData->counter >= 0); \
 }
 
 // variations of GLASS_POOL_ENTER/GLASS_POOL_EXIT allowing them to be used accross different calls

Original file line number	Diff line number	Diff line change
`@@ -362,6 +362,7 @@ - (void)application:(NSApplication )theApplication openFiles:(NSArray )filenam`
`362`	`362`	`LOG("GlassApplication:application:openFiles");`
`363`	`363`
`364`	`364`	`GET_MAIN_JENV;`
	`365`	`+ NSApplicationDelegateReply reply = NSApplicationDelegateReplySuccess;`
`365`	`366`	`NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];`
`366`	`367`	`{`
`367`	`368`	`NSUInteger count = [filenames count];`
`@@ -371,21 +372,26 @@ - (void)application:(NSApplication )theApplication openFiles:(NSArray )filenam`
`371`	`372`	`}`
`372`	`373`	`jobjectArray files = (*env)->NewObjectArray(env, (jsize)count, stringClass, NULL);`
`373`	`374`	`GLASS_CHECK_EXCEPTION(env);`
`374`		`- for (NSUInteger i=0; i<count; i++)`
`375`		`- {`
`376`		`- NSString *file = [filenames objectAtIndex:i];`
`377`		`- if (file != nil)`
	`375`	`+ if (files != NULL) {`
	`376`	`+ for (NSUInteger i=0; i<count; i++)`
`378`	`377`	`{`
`379`		`- (env)->SetObjectArrayElement(env, files, (jsize)i, (env)->NewStringUTF(env, [file UTF8String]));`
`380`		`- GLASS_CHECK_EXCEPTION(env);`
	`378`	`+ NSString *file = [filenames objectAtIndex:i];`
	`379`	`+ if (file != nil)`
	`380`	`+ {`
	`381`	`+ (env)->SetObjectArrayElement(env, files, (jsize)i, (env)->NewStringUTF(env, [file UTF8String]));`
	`382`	`+ GLASS_CHECK_EXCEPTION(env);`
	`383`	`+ }`
`381`	`384`	`}`
	`385`	`+ (*env)->CallVoidMethod(env, self->jApplication, [GlassHelper ApplicationNotifyOpenFilesMethod], files);`
	`386`	`+ } else {`
	`387`	`+ fprintf(stderr, "NewObjectArray failed in GlassApplication_application\n");`
	`388`	`+ reply = NSApplicationDelegateReplyFailure;`
`382`	`389`	`}`
`383`		`- (*env)->CallVoidMethod(env, self->jApplication, [GlassHelper ApplicationNotifyOpenFilesMethod], files);`
`384`	`390`	`}`
`385`	`391`	`[pool drain];`
`386`	`392`	`GLASS_CHECK_EXCEPTION(env);`
`387`	`393`
`388`		`- [theApplication replyToOpenOrPrint:NSApplicationDelegateReplySuccess];`
	`394`	`+ [theApplication replyToOpenOrPrint:reply];`
`389`	`395`	`}`
`390`	`396`
`391`	`397`	`- (BOOL)application:(NSApplication )theApplication openFile:(NSString )filename`