From fb29170bd91436f14d674673f8097970a1bd8952 Mon Sep 17 00:00:00 2001 From: fuchaohong <1783129294@qq.com> Date: Wed, 10 May 2023 14:22:16 +0800 Subject: [PATCH 1/2] Fix '-Dbundle.pmdk' does not take effect, checknative pmdk shows error --- dev-support/bin/dist-copynativelibs | 2 +- hadoop-common-project/hadoop-common/pom.xml | 2 ++ .../hadoop-common/src/CMakeLists.txt | 31 +++++++++---------- .../org/apache/hadoop/io/nativeio/NativeIO.c | 12 +++---- hadoop-project-dist/pom.xml | 4 +++ 5 files changed, 27 insertions(+), 24 deletions(-) diff --git a/dev-support/bin/dist-copynativelibs b/dev-support/bin/dist-copynativelibs index 95de186e7e729..5db23d522f6e0 100755 --- a/dev-support/bin/dist-copynativelibs +++ b/dev-support/bin/dist-copynativelibs @@ -157,7 +157,7 @@ if [[ -d "${LIB_DIR}" ]]; then bundle_native_lib "${ISALBUNDLE}" "isal.lib" "libisal." "${ISALLIB}" - bundle_native_lib "${PMDKBUNDLE}" "pmdk.lib" "pmdk" "${PMDKLIB}" + bundle_native_lib "${PMDKBUNDLE}" "pmdk.lib" "libpmem." "${PMDKLIB}" fi # Windows diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 6c6d3ec5bf929..7dada3e73c56d 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -709,6 +709,8 @@ false + false + false true false diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index d2ef03645a4ae..0393a7de870f0 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -133,24 +133,21 @@ else (ISAL_LIBRARY) ENDIF(REQUIRE_ISAL) endif (ISAL_LIBRARY) -# Build with PMDK library if -Drequire.pmdk option is specified. -if(REQUIRE_PMDK) - set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) - hadoop_set_find_shared_library_version("1") - find_library(PMDK_LIBRARY - NAMES pmem - PATHS ${CUSTOM_PMDK_LIB} /usr/lib /usr/lib64) - set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES}) - - if(PMDK_LIBRARY) - GET_FILENAME_COMPONENT(HADOOP_PMDK_LIBRARY ${PMDK_LIBRARY} REALPATH) - set(PMDK_SOURCE_FILES ${SRC}/io/nativeio/pmdk_load.c) - else(PMDK_LIBRARY) +#Require PMDK +set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) +hadoop_set_find_shared_library_version("1") +find_library(PMDK_LIBRARY + NAMES pmem + PATHS ${CUSTOM_PMDK_LIB} /usr/lib /usr/lib64) +set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES}) +if(PMDK_LIBRARY) + GET_FILENAME_COMPONENT(HADOOP_PMDK_LIBRARY ${PMDK_LIBRARY} NAME) + set(PMDK_SOURCE_FILES ${SRC}/io/nativeio/pmdk_load.c) +else(PMDK_LIBRARY) + if(REQUIRE_PMDK) MESSAGE(FATAL_ERROR "The required PMDK library is NOT found. PMDK_LIBRARY=${PMDK_LIBRARY}") - endif(PMDK_LIBRARY) -else(REQUIRE_PMDK) - MESSAGE(STATUS "Build without PMDK support.") -endif(REQUIRE_PMDK) + endif(REQUIRE_PMDK) +endif(PMDK_LIBRARY) # Build hardware CRC32 acceleration, if supported on the platform. if(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index 1d7c508d85c76..9ed0e5dfb11f3 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -1639,12 +1639,12 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_pm JNIEXPORT jstring JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_getPmdkLibPath (JNIEnv * env, jclass thisClass) { - jstring libpath = NULL; - - #ifdef HADOOP_PMDK_LIBRARY - libpath = (*env)->NewStringUTF(env, HADOOP_PMDK_LIBRARY); - #endif - return libpath; + if (pmdkLoader == NULL) { + THROW(env, "java/lang/UnsatisfiedLinkError", + "Unavailable: library not loaded yet"); + return (jstring)NULL; + } + return (*env)->NewStringUTF(env, pmdkLoader->libname); } #ifdef __cplusplus diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index 5cec569dfa825..f6c19e7e4ca0d 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -45,6 +45,8 @@ false false + + false false false @@ -336,6 +338,8 @@ --artifactid=${project.artifactId} --isalbundle=${bundle.isal} --isallib=${isal.lib} + --pmdkbundle=${bundle.pmdk} + --pmdklib=${pmdk.lib} --openssllib=${openssl.lib} --opensslbinbundle=${bundle.openssl.in.bin} --openssllibbundle=${bundle.openssl} From c3b14d767f6a029abb094298102edd6610c6560c Mon Sep 17 00:00:00 2001 From: fuchaohong <1783129294@qq.com> Date: Thu, 11 May 2023 09:30:09 +0800 Subject: [PATCH 2/2] fix testcase failed --- .../hadoop-common/src/CMakeLists.txt | 31 ++++++++++--------- .../org/apache/hadoop/io/nativeio/NativeIO.c | 7 +++-- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index 0393a7de870f0..4b2ec859bcd57 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -133,21 +133,24 @@ else (ISAL_LIBRARY) ENDIF(REQUIRE_ISAL) endif (ISAL_LIBRARY) -#Require PMDK -set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) -hadoop_set_find_shared_library_version("1") -find_library(PMDK_LIBRARY - NAMES pmem - PATHS ${CUSTOM_PMDK_LIB} /usr/lib /usr/lib64) -set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES}) -if(PMDK_LIBRARY) - GET_FILENAME_COMPONENT(HADOOP_PMDK_LIBRARY ${PMDK_LIBRARY} NAME) - set(PMDK_SOURCE_FILES ${SRC}/io/nativeio/pmdk_load.c) -else(PMDK_LIBRARY) - if(REQUIRE_PMDK) +# Build with PMDK library if -Drequire.pmdk option is specified. +if(REQUIRE_PMDK) + set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + hadoop_set_find_shared_library_version("1") + find_library(PMDK_LIBRARY + NAMES pmem + PATHS ${CUSTOM_PMDK_LIB} /usr/lib /usr/lib64) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES}) + + if(PMDK_LIBRARY) + GET_FILENAME_COMPONENT(HADOOP_PMDK_LIBRARY ${PMDK_LIBRARY} NAME) + set(PMDK_SOURCE_FILES ${SRC}/io/nativeio/pmdk_load.c) + else(PMDK_LIBRARY) MESSAGE(FATAL_ERROR "The required PMDK library is NOT found. PMDK_LIBRARY=${PMDK_LIBRARY}") - endif(REQUIRE_PMDK) -endif(PMDK_LIBRARY) + endif(PMDK_LIBRARY) +else(REQUIRE_PMDK) + MESSAGE(STATUS "Build without PMDK support.") +endif(REQUIRE_PMDK) # Build hardware CRC32 acceleration, if supported on the platform. if(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index 9ed0e5dfb11f3..5dc3c89d015f8 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -1639,12 +1639,15 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_pm JNIEXPORT jstring JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_getPmdkLibPath (JNIEnv * env, jclass thisClass) { + #if (defined UNIX) && (defined HADOOP_PMDK_LIBRARY) if (pmdkLoader == NULL) { THROW(env, "java/lang/UnsatisfiedLinkError", "Unavailable: library not loaded yet"); - return (jstring)NULL; + } else { + return (*env)->NewStringUTF(env, pmdkLoader->libname); } - return (*env)->NewStringUTF(env, pmdkLoader->libname); + #endif + return (jstring)NULL; } #ifdef __cplusplus