8354686: [AIX] now ubsan is possible

Reviewed-by: mbaesken, clanger

Author: Joachim Kern

make/autoconf/jdk-options.m4

@@ -522,6 +522,16 @@ AC_DEFUN_ONCE([JDKOPT_SETUP_UNDEFINED_BEHAVIOR_SANITIZER],
       $ADDITIONAL_UBSAN_CHECKS"
   UBSAN_CFLAGS="$UBSAN_CHECKS -Wno-stringop-truncation -Wno-format-overflow -Wno-array-bounds -Wno-stringop-overflow -fno-omit-frame-pointer -DUNDEFINED_BEHAVIOR_SANITIZER"
   UBSAN_LDFLAGS="$UBSAN_CHECKS"
+  # On AIX, the llvm_symbolizer is not found out of the box, so we have to provide the
+  # full qualified llvm_symbolizer path in the __ubsan_default_options() function in
+  # make/data/ubsan/ubsan_default_options.c. To get it there we compile our sources
+  # with an additional define LLVM_SYMBOLIZER, which we set here.
+  # To calculate the correct llvm_symbolizer path we can use the location of the compiler, because
+  # their relation is fixed.
+  if test "x$TOOLCHAIN_TYPE" = "xclang" && test "x$OPENJDK_TARGET_OS" = "xaix"; then
+      UBSAN_CFLAGS="$UBSAN_CFLAGS -fno-sanitize=function,vptr -DLLVM_SYMBOLIZER=$(dirname $(dirname $CC))/tools/ibm-llvm-symbolizer"
+      UBSAN_LDFLAGS="$UBSAN_LDFLAGS -fno-sanitize=function,vptr -Wl,-bbigtoc"
+  fi
   UTIL_ARG_ENABLE(NAME: ubsan, DEFAULT: false, RESULT: UBSAN_ENABLED,
       DESC: [enable UndefinedBehaviorSanitizer],
       CHECK_AVAILABLE: [

make/data/ubsan/ubsan_default_options.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -43,12 +43,24 @@
 #define ATTRIBUTE_USED
 #endif
 
+// On AIX, the llvm_symbolizer is not found out of the box, so we have to provide the
+// full qualified llvm_symbolizer path in the __ubsan_default_options() function.
+// To get it here we compile our sources with an additional define LLVM_SYMBOLIZER
+// containing the path, which we set in make/autoconf/jdk-options.m4.
+#ifdef LLVM_SYMBOLIZER
+#define _LLVM_SYMBOLIZER(X) ",external_symbolizer_path=" X_LLVM_SYMBOLIZER(X)
+#define X_LLVM_SYMBOLIZER(X) #X
+#else
+#define LLVM_SYMBOLIZER
+#define _LLVM_SYMBOLIZER(X)
+#endif
+
 // Override weak symbol exposed by UBSan to override default options. This is called by UBSan
 // extremely early during library loading, before main is called. We need to override the default
 // options because by default UBSan only prints a warning for each occurrence. We want jtreg tests
 // to fail when undefined behavior is encountered. We also want a full stack trace for the offending
 // thread so it is easier to track down. You can override these options by setting the environment
 // variable UBSAN_OPTIONS.
 ATTRIBUTE_DEFAULT_VISIBILITY ATTRIBUTE_USED const char* __ubsan_default_options() {
-  return "halt_on_error=1,print_stacktrace=1";
+  return "halt_on_error=1,print_stacktrace=1" _LLVM_SYMBOLIZER(LLVM_SYMBOLIZER);
 }