Merge pull request #599 from ibmruntimes/openj9-staging

Merge 11.0.18+5 changes from openj9-staging branch

.github/workflows/submit.yml

@@ -10,7 +10,7 @@ on:
       platforms:
         description: "Platform(s) to execute on"
         required: true
-        default: "Linux additional (hotspot only), Linux x64, Linux x86, Windows x64, macOS x64, macOS aarch64"
+        default: "Linux additional (hotspot only), Linux x64, Linux x86, Windows aarch64, Windows x64, macOS x64, macOS aarch64"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -27,6 +27,7 @@ jobs:
       platform_linux_additional: ${{ steps.check_platforms.outputs.platform_linux_additional }}
       platform_linux_x64: ${{ steps.check_platforms.outputs.platform_linux_x64 }}
       platform_linux_x86: ${{ steps.check_platforms.outputs.platform_linux_x86 }}
+      platform_windows_aarch64: ${{ steps.check_platforms.outputs.platform_windows_aarch64 }}
       platform_windows_x64: ${{ steps.check_platforms.outputs.platform_windows_x64 }}
       platform_macos_x64: ${{ steps.check_platforms.outputs.platform_macos_x64 }}
       platform_macos_aarch64: ${{ steps.check_platforms.outputs.platform_macos_aarch64 }}
@@ -43,6 +44,7 @@ jobs:
           echo "platform_linux_additional=${{ contains(github.event.inputs.platforms, 'linux additional (hotspot only)') || (github.event.inputs.platforms == '' && (secrets.JDK_SUBMIT_PLATFORMS == '' || contains(secrets.JDK_SUBMIT_PLATFORMS, 'linux additional (hotspot only)'))) }}" >> $GITHUB_OUTPUT
           echo "platform_linux_x64=${{ contains(github.event.inputs.platforms, 'linux x64') || (github.event.inputs.platforms == '' && (secrets.JDK_SUBMIT_PLATFORMS == '' || contains(secrets.JDK_SUBMIT_PLATFORMS, 'linux x64'))) }}" >> $GITHUB_OUTPUT
           echo "platform_linux_x86=${{ contains(github.event.inputs.platforms, 'linux x86') || (github.event.inputs.platforms == '' && (secrets.JDK_SUBMIT_PLATFORMS == '' || contains(secrets.JDK_SUBMIT_PLATFORMS, 'linux x86'))) }}" >> $GITHUB_OUTPUT
+          echo "platform_windows_x64=${{ contains(github.event.inputs.platforms, 'windows aarch64') || (github.event.inputs.platforms == '' && (secrets.JDK_SUBMIT_PLATFORMS == '' || contains(secrets.JDK_SUBMIT_PLATFORMS, 'windows aarch64'))) }}" >> $GITHUB_OUTPUT
           echo "platform_windows_x64=${{ contains(github.event.inputs.platforms, 'windows x64') || (github.event.inputs.platforms == '' && (secrets.JDK_SUBMIT_PLATFORMS == '' || contains(secrets.JDK_SUBMIT_PLATFORMS, 'windows x64'))) }}" >> $GITHUB_OUTPUT
           echo "platform_macos_x64=${{ contains(github.event.inputs.platforms, 'macos x64') || (github.event.inputs.platforms == '' && (secrets.JDK_SUBMIT_PLATFORMS == '' || contains(secrets.JDK_SUBMIT_PLATFORMS, 'macos x64'))) }}" >> $GITHUB_OUTPUT
           echo "platform_macos_aarch64=${{ contains(github.event.inputs.platforms, 'macos aarch64') || (github.event.inputs.platforms == '' && (secrets.JDK_SUBMIT_PLATFORMS == '' || contains(secrets.JDK_SUBMIT_PLATFORMS, 'macos aarch64'))) }}" >> $GITHUB_OUTPUT
@@ -1058,6 +1060,106 @@ jobs:
           path: ~/linux-x86${{ matrix.artifact }}_testsupport_${{ env.logsuffix }}.zip
         continue-on-error: true
 
+  windows_aarch64_build:
+    name: Windows aarch64
+    runs-on: "windows-2019"
+    needs: prerequisites
+    if: needs.prerequisites.outputs.should_run != 'false' && needs.prerequisites.outputs.platform_windows_aarch64 != 'false'
+
+    strategy:
+      fail-fast: false
+      matrix:
+        flavor:
+          - build debug
+        include:
+          - flavor: build debug
+            flags: --enable-debug
+            artifact: -debug
+
+    env:
+      JDK_VERSION: "${{ fromJson(needs.prerequisites.outputs.dependencies).DEFAULT_VERSION_FEATURE }}"
+      BOOT_JDK_VERSION: "${{ fromJson(needs.prerequisites.outputs.dependencies).BOOT_JDK_VERSION }}"
+      BOOT_JDK_FILENAME: "${{ fromJson(needs.prerequisites.outputs.dependencies).WINDOWS_X64_BOOT_JDK_FILENAME }}"
+      BOOT_JDK_URL: "${{ fromJson(needs.prerequisites.outputs.dependencies).WINDOWS_X64_BOOT_JDK_URL }}"
+      BOOT_JDK_SHA256: "${{ fromJson(needs.prerequisites.outputs.dependencies).WINDOWS_X64_BOOT_JDK_SHA256 }}"
+
+    steps:
+      - name: Restore cygwin installer from cache
+        id: cygwin-installer
+        uses: actions/cache@v3
+        with:
+          path: ~/cygwin/setup-x86_64.exe
+          key: cygwin-installer
+
+      - name: Download cygwin installer
+        run: |
+          New-Item -Force -ItemType directory -Path "$HOME\cygwin"
+          & curl -L "https://www.cygwin.com/setup-x86_64.exe" -o "$HOME/cygwin/setup-x86_64.exe"
+        if: steps.cygwin-installer.outputs.cache-hit != 'true'
+
+      - name: Restore cygwin packages from cache
+        id: cygwin
+        uses: actions/cache@v3
+        with:
+          path: ~/cygwin/packages
+          key: cygwin-packages-${{ runner.os }}-v1
+
+      - name: Install cygwin
+        run: |
+          Start-Process -FilePath "$HOME\cygwin\setup-x86_64.exe" -ArgumentList "--quiet-mode --packages cygwin=3.3.5-1,autoconf,make,zip,unzip --root $HOME\cygwin\cygwin64 --local-package-dir $HOME\cygwin\packages --site http://mirrors.kernel.org/sourceware/cygwin --no-desktop --no-shortcuts --no-startmenu --no-admin" -Wait -NoNewWindow
+
+      - name: Checkout the source
+        uses: actions/checkout@v3
+        with:
+          path: jdk
+
+      - name: Restore boot JDK from cache
+        id: bootjdk
+        uses: actions/cache@v3
+        with:
+          path: ~/bootjdk/${{ env.BOOT_JDK_VERSION }}
+          key: bootjdk-${{ runner.os }}-${{ env.BOOT_JDK_VERSION }}-${{ env.BOOT_JDK_SHA256 }}-v1
+
+      - name: Download boot JDK
+        run: |
+          mkdir -p "$HOME\bootjdk\$env:BOOT_JDK_VERSION"
+          & curl -L "$env:BOOT_JDK_URL" -o "$HOME/bootjdk/$env:BOOT_JDK_FILENAME"
+          $FileHash = Get-FileHash -Algorithm SHA256 "$HOME/bootjdk/$env:BOOT_JDK_FILENAME"
+          $FileHash.Hash -eq $env:BOOT_JDK_SHA256
+          & tar -xf "$HOME/bootjdk/$env:BOOT_JDK_FILENAME" -C "$HOME/bootjdk/$env:BOOT_JDK_VERSION"
+          Get-ChildItem "$HOME\bootjdk\$env:BOOT_JDK_VERSION\*\*" | Move-Item -Destination "$HOME\bootjdk\$env:BOOT_JDK_VERSION"
+        if: steps.bootjdk.outputs.cache-hit != 'true'
+
+      - name: Ensure a specific version of MSVC is installed
+        run: >
+          Start-Process -FilePath 'C:\Program Files (x86)\Microsoft Visual Studio\Installer\vs_installer.exe' -Wait -NoNewWindow -ArgumentList
+          'modify --installPath "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise" --quiet
+          --add Microsoft.VisualStudio.Component.VC.14.29.arm64'
+
+      - name: Configure
+        run: >
+          $env:Path = "$HOME\cygwin\cygwin64\bin;$HOME\cygwin\cygwin64\bin;$env:Path" ;
+          $env:Path = $env:Path -split ";" -match "C:\\Windows|PowerShell|cygwin" -join ";" ;
+          $env:BOOT_JDK = cygpath "$HOME/bootjdk/$env:BOOT_JDK_VERSION" ;
+          & bash configure
+          --with-conf-name=windows-aarch64
+          --with-msvc-toolset-version=14.29
+          --openjdk-target=aarch64-unknown-cygwin
+          ${{ matrix.flags }}
+          --with-version-opt="$env:GITHUB_ACTOR-$env:GITHUB_SHA"
+          --with-version-build=0
+          --with-boot-jdk="$env:BOOT_JDK"
+          --with-default-make-target="hotspot"
+          --with-jvm-features=shenandoahgc
+        working-directory: jdk
+
+      - name: Build
+        run: |
+          $env:Path = "$HOME\cygwin\cygwin64\bin;$HOME\cygwin\cygwin64\bin;$env:Path" ;
+          $env:Path = $env:Path -split ";" -match "C:\\Windows|PowerShell|cygwin" -join ";" ;
+          & make CONF_NAME=windows-aarch64
+        working-directory: jdk
+
   windows_x64_build:
     name: Windows x64
     runs-on: "windows-2019"
@@ -1880,6 +1982,7 @@ jobs:
     needs:
       - prerequisites
       - linux_additional_build
+      - windows_aarch64_build
       - linux_x64_test
       - linux_x86_test
       - windows_x64_test

closed/CopySupport.gmk

@@ -1,5 +1,5 @@
 # ===========================================================================
-# (c) Copyright IBM Corp. 2020, 2020 All Rights Reserved
+# (c) Copyright IBM Corp. 2020, 2022 All Rights Reserved
 # ===========================================================================
 # This code is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License version 2 only, as
@@ -110,11 +110,11 @@ openj9_copy_files_and_debuginfos = \
 
 # openj9_copy_exes
 # ----------------
-# $1 = list of executable names without $(EXE_SUFFIX)
+# $1 = list of executable names without $(EXECUTABLE_SUFFIX)
 openj9_copy_exes = \
 	$(foreach file, $1, \
 		$(call openj9_copy_files_and_debuginfos, \
-			$(addsuffix /$(file)$(EXE_SUFFIX), \
+			$(addsuffix /$(file)$(EXECUTABLE_SUFFIX), \
 				$(OPENJ9_VM_BUILD_DIR) \
 				$(EXE_DST_DIR))))
 

closed/DDR.gmk

@@ -98,8 +98,10 @@ DDR_COMPATIBILITY_FILE := $(DDR_VM_SRC_ROOT)/com/ibm/j9ddr/CompatibilityConstant
 # When StructureReader opens the blob, it must be able to find AuxFieldInfo29.dat
 # and StructureAliases*.dat, so they must be on the classpath for a non-openj9
 # bootjdk. When using an openj9 bootjdk, we don't want to use old versions that
-# might be included: Patching openj9.dtfj fixes that.
-DDR_TOOLS_PATHLIST := $(call PathList, $(DDR_TOOLS_BIN) $(DDR_VM_SRC_ROOT))
+# might be included: Patching openj9.dtfj fixes that. We use FixPath because
+# fixpath.sh would only fix the first entry of the path.
+DDR_PATH_SEP := $(if $(filter $(OPENJDK_BUILD_OS),windows),;,:)
+DDR_TOOLS_PATHLIST := "$(call FixPath,$(DDR_TOOLS_BIN))$(DDR_PATH_SEP)$(call FixPath,$(DDR_VM_SRC_ROOT))"
 DDR_TOOLS_OPTIONS := \
 	-cp $(DDR_TOOLS_PATHLIST) \
 	--patch-module=openj9.dtfj=$(DDR_TOOLS_PATHLIST)

closed/OpenJ9.gmk

@@ -105,11 +105,10 @@ endif
 
 ifeq (windows,$(OPENJDK_TARGET_OS))
   # convert windows path to unix path
-  UnixPath = $(shell $(CYGPATH) -u $1)
+  UnixPath = $(shell $(PATHTOOL) -u $1)
   # set Visual Studio environment
   # wrap PATH in quotes as it contains spaces (unix path)
-  # INCLUDE, LIB are already wrapped in quotes (windows paths)
-  EXPORT_COMPILER_ENV_VARS := PATH="$(PATH)" INCLUDE=$(INCLUDE) LIB=$(LIB)
+  EXPORT_COMPILER_ENV_VARS := PATH="$(PATH)"
 else ifeq (zos,$(OPENJDK_TARGET_OS))
   UnixPath = $1
   # Allow options to follow the input file name
@@ -156,7 +155,7 @@ define openj9_test_image_rules
 endef
 
 $(foreach file, \
-	$(patsubst %, $(OPENJ9_VM_BUILD_DIR)/%$(EXE_SUFFIX), \
+	$(patsubst %, $(OPENJ9_VM_BUILD_DIR)/%$(EXECUTABLE_SUFFIX), \
 		algotest \
 		bcvunit \
 		cfdump \

closed/autoconf/custom-hook.m4

@@ -156,7 +156,7 @@ AC_DEFUN([OPENJ9_CONFIGURE_COMPILERS],
     [OPENJ9_DEVELOPER_DIR=$with_openj9_developer_dir],
     [OPENJ9_DEVELOPER_DIR=])
   if test "x$OPENJDK_BUILD_OS" = xwindows ; then
-     UTIL_REQUIRE_PROGS([OPENJ9_CLANG], [clang])
+    UTIL_REQUIRE_PROGS([OPENJ9_CLANG], [clang])
   fi
 
   AC_SUBST(OPENJ9_CC)
@@ -174,7 +174,7 @@ AC_DEFUN([OPENJ9_CONFIGURE_CUDA],
       if test -f "$cuda_home/include/cuda.h" ; then
         if test "x$OPENJDK_BUILD_OS_ENV" = xwindows.cygwin ; then
           # UTIL_FIXUP_PATH yields a Unix-style path, but we need a mixed-mode path
-          cuda_home="`$CYGPATH -m $cuda_home`"
+          cuda_home="`$PATHTOOL -m $cuda_home`"
         fi
         if test "$cuda_home" = "$with_cuda" ; then
           AC_MSG_RESULT([$with_cuda])
@@ -196,7 +196,7 @@ AC_DEFUN([OPENJ9_CONFIGURE_CUDA],
       if test -f "$gdk_home/include/nvml.h" ; then
         if test "x$OPENJDK_BUILD_OS_ENV" = xwindows.cygwin ; then
           # UTIL_FIXUP_PATH yields a Unix-style path, but we need a mixed-mode path
-          gdk_home="`$CYGPATH -m $gdk_home`"
+          gdk_home="`$PATHTOOL -m $gdk_home`"
         fi
         if test "$gdk_home" = "$with_gdk" ; then
           AC_MSG_RESULT([$with_gdk])
@@ -287,7 +287,7 @@ AC_DEFUN([OPENJ9_CONFIGURE_HEALTHCENTER],
         else
           if test "x$OPENJDK_BUILD_OS_ENV" = xwindows.cygwin ; then
             # UTIL_FIXUP_PATH yields a Unix-style path, but we need a mixed-mode path
-            healthcenter_jar="`$CYGPATH -m $healthcenter_jar`"
+            healthcenter_jar="`$PATHTOOL -m $healthcenter_jar`"
           fi
           if test "$healthcenter_jar" = "$with_healthcenter" ; then
             AC_MSG_RESULT([$with_healthcenter])
@@ -571,7 +571,7 @@ AC_DEFUN([OPENJ9_THIRD_PARTY_REQUIREMENTS],
     fi
 
     if test "x$OPENJDK_BUILD_OS_ENV" = xwindows.cygwin ; then
-      FREEMARKER_JAR=`$CYGPATH -m "$with_freemarker_jar"`
+      FREEMARKER_JAR=`$PATHTOOL -m "$with_freemarker_jar"`
     else
       FREEMARKER_JAR=$with_freemarker_jar
     fi
@@ -597,6 +597,13 @@ AC_DEFUN_ONCE([CUSTOM_LATE_HOOK],
     OPENJ9_TOOL_DIR="$OUTPUTDIR/tools"
     AC_SUBST(OPENJ9_TOOL_DIR)
     OPENJ9_GENERATE_TOOL_WRAPPERS
+
+    # We used to rely on VS_INCLUDE and VS_LIB directly, but those are no longer available
+    # for substitutions, and they're not Windows-style paths: Convert them for our use.
+    OPENJ9_VS_INCLUDE=`$PATHTOOL -p -w "$VS_INCLUDE"`
+    OPENJ9_VS_LIB=`$PATHTOOL -p -w "$VS_LIB"`
+    AC_SUBST(OPENJ9_VS_INCLUDE)
+    AC_SUBST(OPENJ9_VS_LIB)
   fi
   AC_SUBST(SYSROOT)
   AC_CONFIG_FILES([$OUTPUTDIR/toolchain.cmake:$CLOSED_AUTOCONF_DIR/toolchain.cmake.in])
@@ -750,39 +757,36 @@ AC_DEFUN([CONFIGURE_OPENSSL],
 ])
 
 # Create a tool wrapper for use by cmake.
-# Consists of a shell script which wraps commands with an invocation of a wrapper command.
-# OPENJ9_GENERATE_TOOL_WRAPPER(<name_of_output>, <name_of_wrapper>, <command_to_call>)
+# Consists of a shell script which wraps commands with an invocation of fixpath.
+# OPENJ9_GENERATE_TOOL_WRAPPER(<name_of_wrapper>, <command_to_call>)
 AC_DEFUN([OPENJ9_GENERATE_TOOL_WRAPPER],
 [
   tool_file="$OPENJ9_TOOL_DIR/$1"
 
-  echo "#!/bin/sh" > $tool_file
-  # We need to insert an empty string ([]), to stop M4 treating "$@" as a
-  # variable reference
-  printf '%s "%s" "$[]@"\n' "$2" "$3" >> $tool_file
+  # Separate $ and @ so m4 won't see a variable reference.
+  printf '#!/bin/sh\n%s "$''@"\n' "$2" > $tool_file
   chmod +x $tool_file
 ])
 
 # Generate all the tool wrappers required for cmake on windows
 AC_DEFUN([OPENJ9_GENERATE_TOOL_WRAPPERS],
 [
-  MSVC_BIN_DIR=$($DIRNAME $CC)
-  SDK_BIN_DIR=$($DIRNAME $RC)
-  FIXPATH2="$TOPDIR/closed/fixpath2.sh"
-
   mkdir -p "$OPENJ9_TOOL_DIR"
-  OPENJ9_GENERATE_TOOL_WRAPPER([cl], [$FIXPATH2], [$CC])
-  OPENJ9_GENERATE_TOOL_WRAPPER([clang], [$FIXPATH2], [$OPENJ9_CLANG])
-  OPENJ9_GENERATE_TOOL_WRAPPER([lib], [$FIXPATH2], [$AR])
-  OPENJ9_GENERATE_TOOL_WRAPPER([link], [$FIXPATH2], [$LD])
-  OPENJ9_GENERATE_TOOL_WRAPPER([mc], [$FIXPATH2], [$SDK_BIN_DIR/mc])
-  OPENJ9_GENERATE_TOOL_WRAPPER([ml], [$FIXPATH2], [$MSVC_BIN_DIR/ml])
-  OPENJ9_GENERATE_TOOL_WRAPPER([ml64], [$FIXPATH2], [$MSVC_BIN_DIR/ml64])
-  OPENJ9_GENERATE_TOOL_WRAPPER([nasm], [$FIXPATH2], [$NASM])
-  OPENJ9_GENERATE_TOOL_WRAPPER([rc], [$FIXPATH2], [$RC])
-
-  # fixpath2 can't handle classpaths, or @<file_name> arguments
-  OPENJ9_GENERATE_TOOL_WRAPPER([jar], [$FIXPATH], [$JAR])
-  OPENJ9_GENERATE_TOOL_WRAPPER([java], [$FIXPATH], [$JAVA])
-  OPENJ9_GENERATE_TOOL_WRAPPER([javac], [$FIXPATH],[$JAVAC])
+
+  UTIL_REQUIRE_TOOLCHAIN_PROGS(MC, mc)
+  # Note: the assembler found by OpenJDK macros is 'ml', which is the 32-bit assembler.
+  UTIL_REQUIRE_TOOLCHAIN_PROGS(ML64, ml64)
+
+  OPENJ9_GENERATE_TOOL_WRAPPER([cl], [$CC])
+  OPENJ9_GENERATE_TOOL_WRAPPER([clang], [$OPENJ9_CLANG])
+  OPENJ9_GENERATE_TOOL_WRAPPER([jar], [$JAR])
+  OPENJ9_GENERATE_TOOL_WRAPPER([java], [$JAVA])
+  OPENJ9_GENERATE_TOOL_WRAPPER([javac], [$JAVAC])
+  OPENJ9_GENERATE_TOOL_WRAPPER([lib], [$AR])
+  OPENJ9_GENERATE_TOOL_WRAPPER([link], [$LD])
+  OPENJ9_GENERATE_TOOL_WRAPPER([mc], [$MC])
+  OPENJ9_GENERATE_TOOL_WRAPPER([ml], [$AS])
+  OPENJ9_GENERATE_TOOL_WRAPPER([ml64], [$ML64])
+  OPENJ9_GENERATE_TOOL_WRAPPER([nasm], [$NASM])
+  OPENJ9_GENERATE_TOOL_WRAPPER([rc], [$RC])
 ])

closed/autoconf/custom-spec.gmk.in

@@ -94,8 +94,8 @@ export ac_cv_prog_CXX   := @CXX@
 ifeq ($(OPENJDK_TARGET_OS), windows)
   # Set environment variables for Microsoft Visual Studio toolchain.
   # Note that PATH is set in spec.gmk.
-  export INCLUDE := "@VS_INCLUDE@"
-  export LIB := "@VS_LIB@"
+  export INCLUDE := @OPENJ9_VS_INCLUDE@
+  export LIB := @OPENJ9_VS_LIB@
   export MSVC_VERSION := @TOOLCHAIN_VERSION@
 endif
 

closed/openjdk-tag.gmk

@@ -1 +1 @@
-OPENJDK_TAG := jdk-11.0.18+4
+OPENJDK_TAG := jdk-11.0.18+5

closed/openssl.gmk

@@ -31,7 +31,7 @@ ifeq ($(OPENJDK_TARGET_OS), windows)
   # paths with backslashes; CONFIGURE_INSIST bypasses that requirement.
   # PERL must be an absolute path that will be usable by nmake (i.e. start with
   # a drive letter and a colon).
-  OPENSSL_CONFIG_SETUP := export CONFIGURE_INSIST=true PERL='$(shell $(CYGPATH) -m $(PERL))' &&
+  OPENSSL_CONFIG_SETUP := export CONFIGURE_INSIST=true PERL='$(shell $(PATHTOOL) -m $(PERL))' &&
 
   # Configure produces a makefile intended for use with nmake.
   OPENSSL_MAKE := nmake