CEL4RO31 support for shared library load, lookup and close

Leverage the new LE CEL4RO31 support to enable zOS 64-bit portlib
to load 31-bit shared libraries, lookup target functions.

Co-authored-by: Babneet Singh <sbabneet@ca.ibm.com>
Signed-off-by: Joran Siu <joransiu@ca.ibm.com>

include_core/omrport.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2020 IBM Corp. and others
+ * Copyright (c) 1991, 2021 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -827,6 +827,12 @@ typedef struct J9ProcessorInfos {
 #define OMRPORT_SLOPEN_NO_LOOKUP_MSG_FOR_NOT_FOUND  4
 #define OMRPORT_SLOPEN_OPEN_EXECUTABLE 8     /* Can be ORed without affecting existing flags. */
 
+#if defined(J9ZOS39064)
+#define OMRPORT_SLOPEN_ATTEMPT_31BIT_OPEN 16 /* Attempt 31-bit DLL load from 64-bit. */
+#define OMRPORT_SL_ZOS_31BIT_TARGET_HIGHTAG 0x3100000000000000l /* High-tag to signify 31-bit target handles and addresses. */
+#define OMRPORT_SL_ZOS_31BIT_TARGET_MASK 0xFFFFFFFF /* Mask used to convert 31-bit tagged handles/addresses to proper values. */
+#endif /* defined(J9ZOS39064) */
+
 #define OMRPORT_ARCH_X86       "x86"
 #define OMRPORT_ARCH_PPC       "ppc" 				/* in line with IBM JDK 1.22 and above for AIX and Linux/PPC */
 #define OMRPORT_ARCH_PPC64     "ppc64"

port/CMakeLists.txt

@@ -93,6 +93,7 @@ if(OMR_OS_ZOS)
 			omrvmem_support_above_bar.s
 			omrvmem_support_below_bar_64.s
 			omript_ttoken64.s
+			omrcel4ro31.c
 		)
 	else()
 		list(APPEND OBJECTS omrvmem_support_below_bar_31.s)

port/port_objects.mk

@@ -55,6 +55,7 @@ ifeq (zos,$(OMR_HOST_OS))
 
   ifeq (1,$(OMR_ENV_DATA64))
     # 64-bit only
+    OBJECTS += omrcel4ro31
     OBJECTS += omrget_large_pages_supported
     OBJECTS += omrget_large_2gb_pages_supported
     OBJECTS += omrvmem_support_above_bar

port/zos390/omrcel4ro31.c

@@ -0,0 +1,198 @@
+/*******************************************************************************
+ * Copyright (c) 2021, 2021 IBM Corp. and others
+ *
+ * This program and the accompanying materials are made available under
+ * the terms of the Eclipse Public License 2.0 which accompanies this
+ * distribution and is available at https://www.eclipse.org/legal/epl-2.0/
+ * or the Apache License, Version 2.0 which accompanies this distribution and
+ * is available at https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * This Source Code may also be made available under the following
+ * Secondary Licenses when the conditions for such availability set
+ * forth in the Eclipse Public License, v. 2.0 are satisfied: GNU
+ * General Public License, version 2 with the GNU Classpath
+ * Exception [1] and GNU General Public License, version 2 with the
+ * OpenJDK Assembly Exception [2].
+ *
+ * [1] https://www.gnu.org/software/classpath/license.html
+ * [2] http://openjdk.java.net/legal/assembly-exception.html
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 OR LicenseRef-GPL-2.0 WITH Assembly-exception
+ *******************************************************************************/
+
+#include "omrcel4ro31.h"
+
+#include <unistd.h> /* for _a2e_l */
+#include <string.h> /* for strncpy */
+#include <stdlib.h> /* for malloc31 */
+
+
+/* Function descriptor of CEL4RO31 runtime call from GTCA control block */
+typedef void cel4ro31_cwi_func(void *);
+#define CEL4RO31_FNPTR ((cel4ro31_cwi_func *)((char *)(*(int *)(((char *)__gtca())+1096))+8))
+
+/* Function descriptor of CELQGIPB runtime call from GTCA control block */
+typedef void celqgipb_cwi_func(uint32_t *, OMR_CEL4RO31_infoBlock **, uint32_t *);
+#define CELQGIPB_FNPTR ((celqgipb_cwi_func *)((char *)(*(int *)(((char *)__gtca())+1096))+48))
+
+OMR_CEL4RO31_infoBlock *
+omr_cel4ro31_init(uint32_t flags, const char *moduleName, const char *functionName, uint32_t argsLength)
+{
+	/* Interprocedural buffer (IPB) contains the below-the-bar control blocks and outgoing
+	 * arguments to facilitate the cross-AMODE call.
+	 */
+	uint32_t IPBlength = 0;
+	uint32_t IPBretcode = CELQGIPB_RETCODE_INITIAL;
+	uint32_t moduleNameLength = 0;
+	uint32_t functionNameLength = 0;
+
+	OMR_CEL4RO31_infoBlock *infoBlock = NULL;
+	OMR_CEL4RO31_controlBlock *controlBlock = NULL;
+	OMR_CEL4RO31_module *moduleBlock = NULL;
+	OMR_CEL4RO31_function *functionBlock = NULL;
+
+	/* For DLL load operations, we expect a module name specified. */
+	if (OMR_ARE_ANY_BITS_SET(flags, OMR_CEL4RO31_FLAG_LOAD_DLL)) {
+		moduleNameLength = strlen(moduleName);
+	}
+
+	 /* For DLL query operations, we expect a function name specified. */
+	if (OMR_ARE_ANY_BITS_SET(flags, OMR_CEL4RO31_FLAG_QUERY_TARGET_FUNC)) {
+		functionNameLength = strlen(functionName);
+	}
+
+	/* Request LE-managed IPB in below-the-bar storage. */
+	IPBlength = sizeof(OMR_CEL4RO31_infoBlock) + moduleNameLength + functionNameLength + argsLength +
+	            sizeof(moduleBlock->length) + sizeof(functionBlock->length);
+	if (NULL != CELQGIPB_FNPTR) {
+		CELQGIPB_FNPTR(&IPBlength, &infoBlock, &IPBretcode);
+	}
+
+	if ((CELQGIPB_RETCODE_OK != IPBretcode)) {
+		/* IPB unavailable or request failed. Attempt to allocate below-the-bar storage ourselves. */
+		infoBlock = (OMR_CEL4RO31_infoBlock *)__malloc31(IPBlength);
+		if (NULL == infoBlock) {
+			return NULL;
+		}
+		infoBlock->flags = OMR_CEL4RO31_INFOBLOCK_FLAGS_MALLOC31;
+	} else {
+		infoBlock->flags = 0;
+	}
+
+	controlBlock = &(infoBlock->ro31ControlBlock);
+
+	/* Initialize the RO31 control block members and calculate the various offsets. */
+	controlBlock->version = 1;
+	controlBlock->flags = flags;
+	controlBlock->moduleOffset = sizeof(OMR_CEL4RO31_controlBlock);
+	if (0 == moduleNameLength) {
+		controlBlock->functionOffset = controlBlock->moduleOffset;
+	} else {
+		controlBlock->functionOffset = controlBlock->moduleOffset + moduleNameLength + sizeof(moduleBlock->length);
+	}
+	controlBlock->retcode = 0;
+	/* Control block length should exclude the infoBlock's flags and reference pointers. */
+	controlBlock->length = IPBlength - (sizeof(OMR_CEL4RO31_infoBlock) - sizeof(OMR_CEL4RO31_controlBlock));
+
+	/* For execute target function operations, we need to ensure args reference is set
+	 * up appropriately. If no arguments, argumentsOffset needs to be 0.
+	 */
+	if (0 == argsLength) {
+		controlBlock->argumentsOffset = 0;
+	} else if (0 == functionNameLength) {
+		controlBlock->argumentsOffset = controlBlock->functionOffset;
+	} else {
+		controlBlock->argumentsOffset = controlBlock->functionOffset + functionNameLength + sizeof(functionBlock->length);
+	}
+
+	/* For DLL load operations, we expect a module name specified. */
+	moduleBlock = (OMR_CEL4RO31_module *)((char *)(controlBlock) + controlBlock->moduleOffset);
+	infoBlock->ro31Module = moduleBlock;
+	if (OMR_ARE_ANY_BITS_SET(flags, OMR_CEL4RO31_FLAG_LOAD_DLL)) {
+		moduleBlock->length = moduleNameLength;
+		strncpy(moduleBlock->moduleName, moduleName, moduleNameLength);
+		__a2e_l((char *)moduleBlock->moduleName, moduleNameLength);
+	}
+
+	/* For DLL query operations, we expect a function name specified. */
+	functionBlock = (OMR_CEL4RO31_function *)((char *)(controlBlock) + controlBlock->functionOffset);
+	infoBlock->ro31Function = functionBlock;
+	if (OMR_ARE_ANY_BITS_SET(flags, OMR_CEL4RO31_FLAG_QUERY_TARGET_FUNC)) {
+		functionBlock->length = functionNameLength;
+		strncpy(functionBlock->functionName, functionName, functionNameLength);
+		__a2e_l((char *)functionBlock->functionName, functionNameLength);
+	}
+
+	/* For execute target function operations, we need to ensure args is set
+	 * up appropriately. If no arguments, argumentsOffset needs to be 0.
+	 */
+	if (OMR_ARE_ANY_BITS_SET(flags, OMR_CEL4RO31_FLAG_EXECUTE_TARGET_FUNC)) {
+		infoBlock->ro31_args = (void *)((char *)(controlBlock) + controlBlock->argumentsOffset);
+	}
+
+	return infoBlock;
+}
+
+void
+omr_cel4ro31_deinit(OMR_CEL4RO31_infoBlock *infoBlock)
+{
+	if (NULL != infoBlock) {
+		/* LE-managed IPB will be automatically re-allocated for subsqeuent calls at same level.
+		 * Free any storage we allocated ourselves due to IPB request failure.
+		 */
+		if (OMR_ARE_ANY_BITS_SET(infoBlock->flags, OMR_CEL4RO31_INFOBLOCK_FLAGS_MALLOC31)) {
+			free(infoBlock);
+		}
+	}
+}
+
+void
+omr_cel4ro31_call(OMR_CEL4RO31_infoBlock *infoBlock)
+{
+	/* The CEL4RO31 function may not be available if the LE PTF is not installed. */
+	if (NULL == CEL4RO31_FNPTR) {
+		infoBlock->ro31ControlBlock.retcode = OMR_CEL4RO31_RETCODE_ERROR_LE_RUNTIME_SUPPORT_NOT_FOUND;
+		return;
+	}
+
+	CEL4RO31_FNPTR(&(infoBlock->ro31ControlBlock));
+}
+
+BOOLEAN
+omr_cel4ro31_is_supported(void)
+{
+	return (NULL != CEL4RO31_FNPTR) && (NULL != CELQGIPB_FNPTR);
+}
+
+const char*
+omr_cel4ro31_get_error_message(int retCode)
+{
+	const char* errorMessage;
+	switch(retCode) {
+	case OMR_CEL4RO31_RETCODE_OK:
+		errorMessage = "No errors detected.";
+		break;
+	case OMR_CEL4RO31_RETCODE_ERROR_MULTITHREAD_INVOCATION:
+		errorMessage = "Multi-threaded invocation not supported.";
+		break;
+	case OMR_CEL4RO31_RETCODE_ERROR_STORAGE_ISSUE:
+		errorMessage = "Storage issue detected in CEL4RO31.";
+		break;
+	case OMR_CEL4RO31_RETCODE_ERROR_FAILED_AMODE31_ENV:
+		errorMessage = "Failed to initialize AMODE31 environment.";
+		break;
+	case OMR_CEL4RO31_RETCODE_ERROR_FAILED_LOAD_TARGET_DLL:
+		errorMessage = "DLL module not found.";
+		break;
+	case OMR_CEL4RO31_RETCODE_ERROR_FAILED_QUERY_TARGET_FUNC:
+		errorMessage = "Target function not found.";
+		break;
+	case OMR_CEL4RO31_RETCODE_ERROR_LE_RUNTIME_SUPPORT_NOT_FOUND:
+		errorMessage = "CEL4RO31 runtime support not found.";
+		break;
+	default:
+		errorMessage = "Unexpected return code.";
+		break;
+	}
+	return errorMessage;
+}

port/zos390/omrcel4ro31.h

@@ -0,0 +1,139 @@
+/*******************************************************************************
+ * Copyright (c) 2021, 2021 IBM Corp. and others
+ *
+ * This program and the accompanying materials are made available under
+ * the terms of the Eclipse Public License 2.0 which accompanies this
+ * distribution and is available at https://www.eclipse.org/legal/epl-2.0/
+ * or the Apache License, Version 2.0 which accompanies this distribution and
+ * is available at https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * This Source Code may also be made available under the following
+ * Secondary Licenses when the conditions for such availability set
+ * forth in the Eclipse Public License, v. 2.0 are satisfied: GNU
+ * General Public License, version 2 with the GNU Classpath
+ * Exception [1] and GNU General Public License, version 2 with the
+ * OpenJDK Assembly Exception [2].
+ *
+ * [1] https://www.gnu.org/software/classpath/license.html
+ * [2] http://openjdk.java.net/legal/assembly-exception.html
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 OR LicenseRef-GPL-2.0 WITH Assembly-exception
+ *******************************************************************************/
+
+#ifndef omrcel4ro31_h
+#define omrcel4ro31_h
+
+#include "omrcomp.h"
+
+/* Flags to control the behaviour of CEL4RO31. Used by RO31_CB->flags */
+#define OMR_CEL4RO31_FLAG_LOAD_DLL               0x80000000
+#define OMR_CEL4RO31_FLAG_QUERY_TARGET_FUNC      0x40000000
+#define OMR_CEL4RO31_FLAG_EXECUTE_TARGET_FUNC    0x20000000
+#define OMR_CEL4RO31_FLAG_ALL_LOAD_QUERY_EXECUTE (OMR_CEL4RO31_FLAG_LOAD_DLL | OMR_CEL4RO31_FLAG_QUERY_TARGET_FUNC | OMR_CEL4RO31_FLAG_EXECUTE_TARGET_FUNC)
+
+/* Return codes for CEL4RO31 */
+#define OMR_CEL4RO31_RETCODE_OK                                   0x0
+#define OMR_CEL4RO31_RETCODE_ERROR_MULTITHREAD_INVOCATION         0x1
+#define OMR_CEL4RO31_RETCODE_ERROR_STORAGE_ISSUE                  0x2
+#define OMR_CEL4RO31_RETCODE_ERROR_FAILED_AMODE31_ENV             0x3
+#define OMR_CEL4RO31_RETCODE_ERROR_FAILED_LOAD_TARGET_DLL         0x4
+#define OMR_CEL4RO31_RETCODE_ERROR_FAILED_QUERY_TARGET_FUNC       0x5
+#define OMR_CEL4RO31_RETCODE_ERROR_LE_RUNTIME_SUPPORT_NOT_FOUND   0x6
+
+/* Return codes for CELQGIPB */
+#define CELQGIPB_RETCODE_INITIAL                0xFFFFFFFF
+#define CELQGIPB_RETCODE_OK                     0x0
+#define CELQGIPB_RETCODE_FAILED_ALLOC           0x1
+#define CELQGIPB_RETCODE_DEPTH_EXCEEDED         0x2
+#define CELQGIPB_RETCODE_OTHER_THREAD_SWITCHING 0x3
+
+/* Flags for OMR_CEL4RO31 buffer status */
+#define OMR_CEL4RO31_INFOBLOCK_FLAGS_MALLOC31    0x1
+
+/* Fixed length Control Block structure CEL4RO31 */
+typedef struct OMR_CEL4RO31_controlBlock {
+	uint32_t version;               /**< (Input) A integer which contains the version of RO31_INFO. */
+	uint32_t length;                /**< (Input) A integer which contains the total length of RO31_INFO. */
+	uint32_t flags;                 /**< (Input) Flags to control the behavior of CEL4RO31. */
+	uint32_t moduleOffset;          /**< (Input) Offset to RO31_module section from start of RO31_CB. Req'd for dll load flag. */
+	uint32_t functionOffset;        /**< (Input) Offset to RO31_function section from start of RO31_CB. Req'd for dll query flag. */
+	uint32_t argumentsOffset;       /**< (Input) Offset to outgoing arguments section from start of RO31_CB. Req'd for function execution flag. */
+	uint32_t dllHandle;             /**< DLL handle of target program (Input) DLL handle if dll query flag. (Output) DLL handle if dll load flag. */
+	uint32_t functionEnv;           /**< Environment of target Program (Input) Environment if function execution flag. (Output) Environment if dll query flag. */
+	uint32_t functionEntryPoint;    /**< Entry point of target program (Input) Entry point if function execution flag. (Output) Entry point if dll query flag. */
+	uint32_t gpr15ReturnValue;      /**< (Output) Return GPR buffer containing 32-bit GPR15 value when target program returned after execution. */
+	uint32_t gpr0ReturnValue;       /**< (Output) Return GPR buffer containing 32-bit GPR0 value when target program returned after execution. */
+	uint32_t gpr1ReturnValue;       /**< (Output) Return GPR buffer containing 32-bit GPR1 value when target program returned after execution. */
+	uint32_t gpr2ReturnValue;       /**< (Output) Return GPR buffer containing 32-bit GPR2 value when target program returned after execution. */
+	uint32_t gpr3ReturnValue;       /**< (Output) Return GPR buffer containing 32-bit GPR3 value when target program returned after execution. */
+	int32_t retcode;                /**< (Output) Return code of CEL4RO31. */
+} OMR_CEL4RO31_controlBlock;
+
+/* Internal struct to map module name definition in control block */
+typedef struct OMR_CEL4RO31_module {
+	uint32_t length;        /**< Length of the module name */
+	char moduleName[1];     /**< Pointer to variable length module name. */
+} OMR_CEL4RO31_module;
+
+/* Internal struct to map function name definition in control block */
+typedef struct OMR_CEL4RO31_function {
+	uint32_t length;        /**< Length of the function name */
+	char functionName[1];   /**< Pointer to variable length function name. */
+} OMR_CEL4RO31_function;
+
+/* Internal struct to track the RO31_CB and related variable length structures */
+typedef struct OMR_CEL4RO31_infoBlock {
+	uint32_t flags;                                  /**< Miscellaneous flags denoting state of buffer. */
+	OMR_CEL4RO31_module *ro31Module;                 /**< Pointer to the start of the module section of the control block. */
+	OMR_CEL4RO31_function *ro31Function;             /**< Pointer to the start of the function section of the control block. */
+	void *ro31_args;                                 /**< Pointer to the start of the outgoing arguments section of the control block. */
+	OMR_CEL4RO31_controlBlock ro31ControlBlock;      /**< Pointer to the control block for CEL4RO31. */
+} OMR_CEL4RO31_infoBlock;
+
+/**
+ * A helper routine to allocate the CEL4RO31 control blocks and related structures
+ * for module, function and arguments.
+ * @param[in] flags The OMR_CEL4RO31_FLAG_* flags for load library, query and execute target functions.
+ * @param[in] moduleName The name of the target library to load.
+ * @param[in] functionName The name of the target function to lookup.
+ * @param[in] argsLength The length of the outgoing parameters, in bytes.
+ *
+ * @return An initialized CEL4RO31 control block, NULL if not successful.
+ */
+OMR_CEL4RO31_infoBlock *
+omr_cel4ro31_init(uint32_t flags, const char* moduleName, const char* functionName, uint32_t argsLength);
+
+/**
+ * A Helper routine to deallocate the CEL4RO31 control blocks and related structures
+ * for module, function and arguments.
+ * @param[in] infoBlock A control block associated with this call for deallocation.
+ */
+void
+omr_cel4ro31_deinit(OMR_CEL4RO31_infoBlock *infoBlock);
+
+/**
+ * A helper routine to make the CEL4RO31 call.
+ * @param[in] infoBlock A completed control block for target invocation.
+ *
+ * @note The success/failure of this call invocation is indicated via the retcode member of the control block.
+ */
+void
+omr_cel4ro31_call(OMR_CEL4RO31_infoBlock *infoBlock);
+
+/**
+ * A helper routine to confirm LE support of CEL4RO31
+ * @return whether CEL4RO31 runtime support is available.
+ */
+BOOLEAN
+omr_cel4ro31_isSupported(void);
+
+/**
+ * A helper routine to return an error message associated with the CEL4RO31 return code.
+ * @param[in] retCode A CEL4RO31 return code.
+ *
+ * @return A message describing the conditions of the given error.
+ */
+const char *
+omr_cel4ro31_get_error_message(int retCode);
+
+#endif /* omrcel4ro31_h */