Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implement COMGR V2 support with cmake -DUSE_COMGR=1 guard #1067

Closed
wants to merge 2 commits into from
Closed

Implement COMGR V2 support with cmake -DUSE_COMGR=1 guard #1067

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
39f7bd2
Implement COMGR V2 support with cmake -DUSE_COMGR=1 guard
aaronenyeshi May 2, 2019
d6b0025
Fix bug in COMGR guard not propagating to hipcc
aaronenyeshi May 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 24 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Expand Up @@ -193,6 +193,25 @@ if(USE_PROF_API EQUAL 1)
endif()
endif()

################
# Detect Code Object Manager
################

if(USE_COMGR EQUAL 1)
if(HIP_PLATFORM STREQUAL "hcc")
find_package(amd_comgr REQUIRED CONFIG
PATHS
/opt/rocm/
PATH_SUFFIXES
cmake/amd_comgr
lib/cmake/amd_comgr
)
add_definitions(-DUSE_COMGR=1)
MESSAGE(STATUS "Code Object Manager found at ${amd_comgr_DIR}.")
endif()
endif()
add_to_config(_buildInfo USE_COMGR)

#############################
# Build steps
#############################
Expand Down Expand Up @@ -264,6 +283,11 @@ if(HIP_PLATFORM STREQUAL "hcc")
target_link_libraries(hip_hcc_static PRIVATE hc_am)
endif()

if(USE_COMGR EQUAL 1)
target_link_libraries(hip_hcc PRIVATE amd_comgr)
target_link_libraries(hip_hcc_static PRIVATE amd_comgr)
endif()

string(REPLACE " " ";" HCC_CXX_FLAGS_LIST ${HCC_CXX_FLAGS})
foreach(TARGET hip_hcc hip_hcc_static)
target_include_directories(${TARGET} SYSTEM INTERFACE $<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>;${HSA_PATH}/include)
Expand Down
15 changes: 15 additions & 0 deletions bin/hipcc
View file
Open in desktop
Expand Up @@ -143,6 +143,16 @@ if ($verbose & 0x2) {
print ("HIP_PLATFORM=$HIP_PLATFORM\n");
}

#---
# Find COMGR location
$USE_COMGR= $hipConfig{'USE_COMGR'};
if ($USE_COMGR eq 1) {
$COMGR_PATH=$ENV{'COMGR_PATH'};
if (!defined $COMGR_PATH) {
$COMGR_PATH = "/opt/rocm";
}
}

# set if user explicitly requests -stdlib=libc++. (else we default to libstdc++ for better interop with g++):
$setStdLib = 0; # TODO - set to 0

Expand Down Expand Up @@ -279,6 +289,11 @@ if ($HIP_PLATFORM eq "clang") {
}
}

if ($USE_COMGR eq 1) {
$HIPCXXFLAGS .= " -DUSE_COMGR=1 -isystem $COMGR_PATH/include";
$HIPLDFLAGS .= " -L$COMGR_PATH/lib -Wl,--rpath=$COMGR_PATH/lib -lamd_comgr ";
}

$marker_lib_path = "$marker_path/bin/x86_64";
if (-e $marker_lib_path) {
$HIPLDFLAGS .= " -L$marker_lib_path -lCXLActivityLogger -Wl,--rpath=$marker_lib_path";
Expand Down
1 change: 1 addition & 0 deletions hip-config-clang.cmake.in
View file
Open in desktop
Expand Up @@ -50,6 +50,7 @@ set_and_check( hip_BIN_INSTALL_DIR "@PACKAGE_BIN_INSTALL_DIR@" )
set_and_check(hip_HIPCC_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipcc")
set_and_check(hip_HIPCONFIG_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipconfig")

find_dependency(amd_comgr)
include( "${CMAKE_CURRENT_LIST_DIR}/hip-targets.cmake" )

set( hip_LIBRARIES hip::host hip::device)
Expand Down
1 change: 1 addition & 0 deletions hip-config-hcc.cmake.in
View file
Open in desktop
Expand Up @@ -51,6 +51,7 @@ set_and_check(hip_HIPCC_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipcc")
set_and_check(hip_HIPCONFIG_EXECUTABLE "${hip_BIN_INSTALL_DIR}/hipconfig")

find_dependency(hcc)
find_dependency(amd_comgr)
include( "${CMAKE_CURRENT_LIST_DIR}/hip-targets.cmake" )

set( hip_LIBRARIES hip::host hip::device)
Expand Down
194 changes: 194 additions & 0 deletions include/hip/hcc_detail/program_state.hpp
View file
Open in desktop
Expand Up @@ -40,6 +40,10 @@ THE SOFTWARE.
#include <hsa/hsa_ext_amd.h>
#include <hsa/hsa_ven_amd_loader.h>

#if USE_COMGR
#include <amd_comgr.h>
#endif

#include <link.h>

#include <cassert>
Expand All @@ -55,6 +59,7 @@ THE SOFTWARE.
#include <unordered_map>
#include <utility>
#include <vector>
#include <iostream>

struct ihipModuleSymbol_t;
using hipFunction_t = ihipModuleSymbol_t*;
Expand Down Expand Up @@ -557,6 +562,190 @@ const std::unordered_map<
return r[agent].second;
}

#if USE_COMGR
inline
void checkError(
amd_comgr_status_t status,
char const *str) {
if (status != AMD_COMGR_STATUS_SUCCESS) {
const char *status_str;
status = amd_comgr_status_string(status, &status_str);
if (status == AMD_COMGR_STATUS_SUCCESS)
std::cerr << "FAILED: " << str << "\n REASON: " << status_str << std::endl;
hip_throw(std::runtime_error{"Metadata parsing failed."});
}
}

class comgr_metadata_node {
public:
amd_comgr_metadata_node_t node;
bool active;
comgr_metadata_node() : active(false) {}
~comgr_metadata_node() {
if(active)
checkError(amd_comgr_destroy_metadata(node), "amd_comgr_destroy_metadata");
}
bool is_active() { return active; }
void set_active(bool value) { active = value; }
comgr_metadata_node(const comgr_metadata_node&) = delete;
comgr_metadata_node(comgr_metadata_node&&) = delete;
comgr_metadata_node& operator=(const comgr_metadata_node&) = delete;
comgr_metadata_node& operator=(comgr_metadata_node&&) = delete;

};

class comgr_data {
public:
amd_comgr_data_t data;
bool active;
comgr_data() : active(false) {}
~comgr_data() {
if(active)
checkError(amd_comgr_release_data(data), "amd_comgr_release_data");
}
bool is_active() { return active; }
void set_active(bool value) { active = value; }
comgr_data(const comgr_data&) = delete;
comgr_data(comgr_data&&) = delete;
comgr_data& operator=(const comgr_data&) = delete;
comgr_data& operator=(comgr_data&&) = delete;

};

inline
std::string lookup_keyword_value(
amd_comgr_metadata_node_t& in_node,
std::string keyword) {
amd_comgr_status_t status;
size_t value_size;
comgr_metadata_node value_meta;

status = amd_comgr_metadata_lookup(in_node, keyword.c_str(), &value_meta.node);
checkError(status, "amd_comgr_metadata_lookup");
value_meta.set_active(true);
status = amd_comgr_get_metadata_string(value_meta.node, &value_size, NULL);
checkError(status, "amd_comgr_get_metadata_string");
// Since value_size returns size with null terminator, we don't include for C++ string size
value_size--;
std::string value(value_size, '\0');
status = amd_comgr_get_metadata_string(value_meta.node, &value_size, &value[0]);
checkError(status, "amd_comgr_get_metadata_string");

return value;
}

inline
void process_kernarg_metadata(
amd_comgr_metadata_node_t blob_meta,
std::unordered_map<
std::string,
std::vector<std::pair<std::size_t, std::size_t>>>& kernargs) {
amd_comgr_status_t status;
amd_comgr_metadata_kind_t mkindLookup;
comgr_metadata_node kernelList;
std::string kernel_name;
size_t num_kernels = 0;
size_t num_kern_args = 0;

// Kernels is a list of MAPS!!
status = amd_comgr_metadata_lookup(blob_meta, "Kernels", &kernelList.node);
// FIXME - few hip memset kernels are missing Kernels node
if(status != AMD_COMGR_STATUS_SUCCESS)
return;
kernelList.set_active(true);

status = amd_comgr_get_metadata_kind(kernelList.node, &mkindLookup);
if (mkindLookup != AMD_COMGR_METADATA_KIND_LIST) {
hip_throw(std::runtime_error{"Lookup of Kernels didn't return a list\n"});
}

status = amd_comgr_get_metadata_list_size(kernelList.node, &num_kernels);
checkError(status, "amd_comgr_get_metadata_list_size");
for (int i = 0; i < num_kernels; i++) {
comgr_metadata_node kernelMap;
status = amd_comgr_index_list_metadata(kernelList.node, i, &kernelMap.node);
checkError(status, "amd_comgr_index_list_metadata");
kernelMap.set_active(true);

kernel_name = std::move(lookup_keyword_value(kernelMap.node, "Name"));

// Check if this kernel was already processed
if(!kernargs[kernel_name].empty()) {
continue;
}

comgr_metadata_node kernArgList;
status = amd_comgr_metadata_lookup(kernelMap.node, "Args", &kernArgList.node);
if (status == AMD_COMGR_STATUS_SUCCESS ) {
kernArgList.set_active(true);
status = amd_comgr_get_metadata_kind(kernArgList.node, &mkindLookup);
if (mkindLookup != AMD_COMGR_METADATA_KIND_LIST) {
hip_throw(std::runtime_error{"Lookup of Args didn't return a list\n"});
}

if (status == AMD_COMGR_STATUS_SUCCESS ) {
status = amd_comgr_get_metadata_list_size(kernArgList.node, &num_kern_args);
checkError(status, "amd_comgr_get_metadata_list_size");
for (int k_ar = 0; k_ar < num_kern_args; k_ar++) {
comgr_metadata_node kernArgMap;
status = amd_comgr_index_list_metadata(kernArgList.node, k_ar, &kernArgMap.node);
checkError(status, "amd_comgr_index_list_metadata");
kernArgMap.set_active(true);
size_t k_arg_size, k_arg_align;

k_arg_size = std::stoul(lookup_keyword_value(kernArgMap.node, "Size"));
k_arg_align = std::stoul(lookup_keyword_value(kernArgMap.node, "Align"));

// Save it into our kernargs
kernargs[kernel_name].emplace_back(k_arg_size, k_arg_align);

} // end kernArgMap
}
} // end kernArgList
} // end kernelMap
} // end kernelList

inline
void read_kernarg_metadata(
std::string blob,
std::unordered_map<
std::string,
std::vector<std::pair<std::size_t, std::size_t>>>& kernargs) {

const char *blob_buf = blob.data();
long blob_size = blob.size();

amd_comgr_status_t status;
comgr_data blob_data;
status = amd_comgr_create_data(AMD_COMGR_DATA_KIND_RELOCATABLE, &blob_data.data);
checkError(status, "amd_comgr_create_data");
blob_data.set_active(true);

status = amd_comgr_set_data(blob_data.data, blob_size, blob_buf);
if(status != AMD_COMGR_STATUS_SUCCESS)
return;

// We have a valid code object now
status = amd_comgr_set_data_name(blob_data.data, "HIP Code Object");
checkError(status, "amd_comgr_set_data_name");

comgr_metadata_node blob_meta;
status = amd_comgr_get_data_metadata(blob_data.data, &blob_meta.node);
checkError(status, "amd_comgr_get_data_metadata");
blob_meta.set_active(true);

// Root is a map
amd_comgr_metadata_kind_t blob_mkind;
status = amd_comgr_get_metadata_kind(blob_meta.node, &blob_mkind);
checkError(status, "amd_comgr_get_metadata_kind");
if (blob_mkind != AMD_COMGR_METADATA_KIND_MAP) {
hip_throw(std::runtime_error{"Root is not map\n"});
}

process_kernarg_metadata(blob_meta.node, kernargs);
}

#else
inline
std::size_t parse_args(
const std::string& metadata,
Expand Down Expand Up @@ -645,6 +834,7 @@ void read_kernarg_metadata(
} while (true);
}
}
#endif

inline
__attribute__((visibility("hidden")))
Expand All @@ -657,13 +847,17 @@ const std::unordered_map<
std::call_once(f, []() {
for (auto&& isa_blobs : code_object_blobs()) {
for (auto&& blob : isa_blobs.second) {
#if USE_COMGR
read_kernarg_metadata(std::string{blob.cbegin(), blob.cend()}, r);
#else
std::stringstream tmp{std::string{blob.cbegin(), blob.cend()}};

ELFIO::elfio reader;

if (!reader.load(tmp)) continue;

read_kernarg_metadata(reader, r);
#endif
}
}
});
Expand Down
8 changes: 4 additions & 4 deletions packaging/hip_hcc.txt
View file
Open in desktop
Expand Up @@ -30,19 +30,19 @@ set(CPACK_GENERATOR "TGZ;DEB;RPM")
set(CPACK_BINARY_DEB "ON")
set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${PROJECT_BINARY_DIR}/postinst;${PROJECT_BINARY_DIR}/prerm")
if(@COMPILE_HIP_ATP_MARKER@)
set(CPACK_DEBIAN_PACKAGE_DEPENDS "hip_base (= ${CPACK_PACKAGE_VERSION}), ${HCC_PACKAGE_NAME} (= @HCC_PACKAGE_VERSION@), rocm-profiler")
set(CPACK_DEBIAN_PACKAGE_DEPENDS "hip_base (= ${CPACK_PACKAGE_VERSION}), ${HCC_PACKAGE_NAME} (= @HCC_PACKAGE_VERSION@), rocm-profiler, comgr (>= 1.1)")
else()
set(CPACK_DEBIAN_PACKAGE_DEPENDS "hip_base (= ${CPACK_PACKAGE_VERSION}), ${HCC_PACKAGE_NAME} (= @HCC_PACKAGE_VERSION@)")
set(CPACK_DEBIAN_PACKAGE_DEPENDS "hip_base (= ${CPACK_PACKAGE_VERSION}), ${HCC_PACKAGE_NAME} (= @HCC_PACKAGE_VERSION@), comgr (>= 1.1)")
endif()
set(CPACK_BINARY_RPM "ON")
set(CPACK_RPM_PACKAGE_ARCHITECTURE "${CMAKE_SYSTEM_PROCESSOR}")
set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${PROJECT_BINARY_DIR}/postinst")
set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${PROJECT_BINARY_DIR}/prerm")
set(CPACK_RPM_PACKAGE_AUTOREQPROV " no")
if(@COMPILE_HIP_ATP_MARKER@)
set(CPACK_RPM_PACKAGE_REQUIRES "hip_base = ${CPACK_PACKAGE_VERSION}, ${HCC_PACKAGE_NAME} = @HCC_PACKAGE_VERSION@, rocm-profiler")
set(CPACK_RPM_PACKAGE_REQUIRES "hip_base = ${CPACK_PACKAGE_VERSION}, ${HCC_PACKAGE_NAME} = @HCC_PACKAGE_VERSION@, rocm-profiler, comgr")
else()
set(CPACK_RPM_PACKAGE_REQUIRES "hip_base = ${CPACK_PACKAGE_VERSION}, ${HCC_PACKAGE_NAME} = @HCC_PACKAGE_VERSION@")
set(CPACK_RPM_PACKAGE_REQUIRES "hip_base = ${CPACK_PACKAGE_VERSION}, ${HCC_PACKAGE_NAME} = @HCC_PACKAGE_VERSION@, comgr")
endif()
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/opt")
set(CPACK_SOURCE_GENERATOR "TGZ")
Expand Down