Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable GDS and Support Runtime Context (__enter__, __exit__) for CuFileDriver and CuImage #106

Merged
merged 4 commits into from
Sep 29, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ include(ExternalProject)
################################################################################
# Options
################################################################################
option(CUCIM_SUPPORT_GDS "Support cufile library" OFF)
option(CUCIM_SUPPORT_GDS "Support cufile library" ON)
option(CUCIM_STATIC_GDS "Use static cufile library" OFF)
option(CUCIM_SUPPORT_CUDA "Support CUDA" ON)

Expand Down
4 changes: 3 additions & 1 deletion cpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ add_library(${CUCIM_PACKAGE_NAME}
include/cucim/memory/memory_manager.h
include/cucim/util/cuda.h
include/cucim/util/file.h
include/cucim/util/platform.h
include/cucim/3rdparty/dlpack/dlpack.h
include/cucim/3rdparty/dlpack/dlpackcpp.h
src/cuimage.cpp
Expand Down Expand Up @@ -95,7 +96,8 @@ add_library(${CUCIM_PACKAGE_NAME}
src/logger/logger.cpp
src/logger/timer.cpp
src/memory/memory_manager.cu
src/util/file.cpp)
src/util/file.cpp
src/util/platform.cpp)

# Compile options
set_target_properties(${CUCIM_PACKAGE_NAME}
Expand Down
2 changes: 2 additions & 0 deletions cpp/include/cucim/cuimage.h
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ class EXPORT_VISIBLE CuImage : public std::enable_shared_from_this<CuImage>

void save(std::string file_path) const;

void close();

private:
using Mutex = std::mutex;
using ScopedLock = std::scoped_lock<Mutex>;
Expand Down
32 changes: 32 additions & 0 deletions cpp/include/cucim/util/platform.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
* Copyright (c) 2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//
#ifndef CUCIM_UTIL_PLATFORM_H
#define CUCIM_UTIL_PLATFORM_H

#include "cucim/macros/api_header.h"

/**
* @brief Platform-specific macros and functions.
*/
namespace cucim::util
{

EXPORT_VISIBLE bool is_in_wsl();

} // namespace cucim::util

#endif // CUCIM_UTIL_PLATFORM_H
25 changes: 21 additions & 4 deletions cpp/src/cuimage.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -229,10 +229,7 @@ CuImage::CuImage() : std::enable_shared_from_this<CuImage>()
CuImage::~CuImage()
{
// printf("[cuCIM] CuImage::~CuImage()\n");
if (file_handle_.client_data)
{
image_formats_->formats[0].image_parser.close(&file_handle_);
}
close();
image_formats_ = nullptr; // memory release is handled by the framework
if (image_metadata_)
{
Expand Down Expand Up @@ -615,6 +612,10 @@ CuImage CuImage::read_region(std::vector<int64_t>&& location,
// Read region from internal file if image_data_ is nullptr
if (image_data_ == nullptr)
{
if (file_handle_.fd < 0) // file_handle_ is not opened
{
throw std::runtime_error("[Error] The image file is closed!");
}
if (!image_formats_->formats[0].image_reader.read(
&file_handle_, image_metadata_, &request, image_data, nullptr /*out_metadata*/))
{
Expand Down Expand Up @@ -778,6 +779,10 @@ std::set<std::string> CuImage::associated_images() const

CuImage CuImage::associated_image(const std::string& name, const io::Device& device) const
{
if (file_handle_.fd < 0) // file_handle_ is not opened
{
throw std::runtime_error("[Error] The image file is closed!");
}
auto it = associated_images_.find(name);
if (it != associated_images_.end())
{
Expand Down Expand Up @@ -855,6 +860,18 @@ void CuImage::save(std::string file_path) const
fs.close();
}
}

void CuImage::close()
{
if (file_handle_.client_data)
{
image_formats_->formats[0].image_parser.close(&file_handle_);
}
file_handle_.cufile = nullptr;
file_handle_.path = nullptr;
file_handle_.fd = -1;
}

void CuImage::ensure_init()
{
ScopedLock g(mutex_);
Expand Down
12 changes: 8 additions & 4 deletions cpp/src/filesystem/cufile_driver.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include <fmt/format.h>

#include "cucim/util/cuda.h"
#include "cucim/util/platform.h"
#include "cufile_stub.h"

#define ALIGN_UP(x, align_to) (((uint64_t)(x) + ((uint64_t)(align_to)-1)) & ~((uint64_t)(align_to)-1))
Expand Down Expand Up @@ -100,7 +101,7 @@ static int get_file_flags(const char* flags)

bool is_gds_available()
{
return static_cast<bool>(s_cufile_initializer);
return static_cast<bool>(s_cufile_initializer) && !cucim::util::is_in_wsl();
}

std::shared_ptr<CuFileDriver> open(const char* file_path, const char* flags, mode_t mode)
Expand Down Expand Up @@ -529,7 +530,8 @@ ssize_t CuFileDriver::pread(void* buf, size_t count, off_t file_offset, off_t bu
{
if (memory_type == cudaMemoryTypeUnregistered)
{
read_cnt = ::pread(handle_.fd, reinterpret_cast<char*>(buf) + buf_offset, block_read_size, file_offset);
read_cnt =
::pread(handle_.fd, reinterpret_cast<char*>(buf) + buf_offset, block_read_size, file_offset);
total_read_cnt += read_cnt;
}
else
Expand Down Expand Up @@ -838,7 +840,8 @@ ssize_t CuFileDriver::pwrite(const void* buf, size_t count, off_t file_offset, o
{
if (memory_type == cudaMemoryTypeUnregistered)
{
write_cnt = ::pwrite(handle_.fd, reinterpret_cast<const char*>(buf) + buf_offset, block_write_size, file_offset);
write_cnt = ::pwrite(
handle_.fd, reinterpret_cast<const char*>(buf) + buf_offset, block_write_size, file_offset);
total_write_cnt += write_cnt;
}
else
Expand Down Expand Up @@ -1100,7 +1103,8 @@ ssize_t CuFileDriver::pwrite(const void* buf, size_t count, off_t file_offset, o
{
(void*)s_cufile_cache.device_cache(); // Lazy initialization

ssize_t write_cnt = cuFileWrite(handle_.cufile, reinterpret_cast<const char*>(buf) + buf_offset, count, file_offset, 0);
ssize_t write_cnt =
cuFileWrite(handle_.cufile, reinterpret_cast<const char*>(buf) + buf_offset, count, file_offset, 0);
if (write_cnt < 0)
{
fmt::print(stderr, "[cuFile Error] {}\n", CUFILE_ERRSTR(write_cnt));
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/filesystem/file_handle.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
#include "cucim/codec/hash_function.h"

CuCIMFileHandle::CuCIMFileHandle()
: fd(0),
: fd(-1),
cufile(nullptr),
type(FileHandleType::kUnknown),
path(nullptr),
Expand Down
16 changes: 16 additions & 0 deletions cpp/src/util/file.cpp
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright (c) 2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "cucim/util/file.h"

#include <sys/stat.h>
Expand Down
46 changes: 46 additions & 0 deletions cpp/src/util/platform.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright (c) 2021, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "cucim/util/platform.h"

#include <stdio.h>
#include <string.h>
#include <sys/utsname.h>


namespace cucim::util
{

bool is_in_wsl()
{
struct utsname buf;
int err = uname(&buf);
if (err == 0)
{
char* pos = strstr(buf.release, "icrosoft");
if (pos)
{
// 'Microsoft' for WSL1 and 'microsoft' for WSL2
if (buf.release < pos && (pos[-1] == 'm' || pos[-1] == 'M'))
{
return true;
}
}
}
return false;
}

} // namespace cucim::util
10 changes: 9 additions & 1 deletion gds/src/cufile_stub.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

#include "cufile_stub.h"
#include "cucim/dynlib/helper.h"
#include "cucim/util/platform.h"

#define IMPORT_FUNCTION(handle, name) impl_##name = cucim::dynlib::get_library_symbol<t_##name>(handle, #name);

Expand Down Expand Up @@ -196,9 +197,16 @@ extern "C"

CUfileError_t cuFileDriverOpen(void)
{
// GDS v1.0.0 does not support WSL and executing this can cause the following error:
// Assertion failure, file index :cufio-udev line :143
// So we do not call impl_cuFileDriverOpen() here if the current platform is WSL.
if (impl_cuFileDriverOpen)
{
return impl_cuFileDriverOpen();
// If not in WSL, call impl_cuFileDriverOpen()
if (!cucim::util::is_in_wsl())
{
return impl_cuFileDriverOpen();
}
}
return CUfileError_t{ CU_FILE_DRIVER_NOT_INITIALIZED, CUDA_SUCCESS };
}
Expand Down
14 changes: 14 additions & 0 deletions python/pybind11/cucim_py.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,21 @@ PYBIND11_MODULE(_cucim, m)
py::arg("name") = "", //
py::arg("device") = io::Device()) //
.def("save", &CuImage::save, doc::CuImage::doc_save, py::call_guard<py::gil_scoped_release>()) //
.def("close", &CuImage::close, doc::CuImage::doc_close, py::call_guard<py::gil_scoped_release>()) //
.def("__bool__", &CuImage::operator bool, py::call_guard<py::gil_scoped_release>()) //
.def(
"__enter__",
[](const std::shared_ptr<CuImage>& cuimg) { //
return cuimg; //
}, //
py::call_guard<py::gil_scoped_release>())
.def(
"__exit__",
[](const std::shared_ptr<CuImage>& cuimg, const py::object& type, const py::object& value,
const py::object& traceback) { //
cuimg->close(); //
}, //
py::call_guard<py::gil_scoped_release>())
.def(
"__repr__", //
[](const CuImage& cuimg) { //
Expand Down
9 changes: 9 additions & 0 deletions python/pybind11/cucim_pydoc.h
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,15 @@ Saves image data to the file path.
Currently it supports only .ppm file format that can be viewed by `eog` command in Ubuntu.
)doc")

// void close();
PYDOC(close, R"doc(
Closes the file handle.

Once the file handle is closed, the image object (if loaded before) still exists but cannot read additional images
from the file.
)doc")


// void _set_array_interface(const CuImage& cuimg);
PYDOC(_set_array_interface, R"doc(
Add `__array_interface__` or `__cuda_array_interface__` depending on the memory type.
Expand Down
13 changes: 13 additions & 0 deletions python/pybind11/filesystem/filesystem_py.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,19 @@ void init_filesystem(py::module& fs)
py::arg("file_offset"), //
py::arg("buf_offset") = 0) //
.def("close", &CuFileDriver::close, doc::CuFileDriver::doc_close, py::call_guard<py::gil_scoped_release>()) //
.def(
"__enter__",
[](const std::shared_ptr<CuFileDriver>& fd) { //
return fd; //
}, //
py::call_guard<py::gil_scoped_release>())
.def(
"__exit__",
[](const std::shared_ptr<CuFileDriver>& fd, const py::object& type, const py::object& value,
const py::object& traceback) { //
fd->close(); //
}, //
py::call_guard<py::gil_scoped_release>())
.def("__repr__", [](const CuFileDriver& fd) {
return fmt::format("<cucim.clara.filesystem.CuFileDriver path:{}>", fd.path());
});
Expand Down
9 changes: 5 additions & 4 deletions run
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,7 @@ build_local() {
copy_gds_files_() {
local root_folder=${1:-${TOP}}
local cufile_search="${root_folder}/temp/cuda/include:${root_folder}/temp/cuda/lib64" #"/usr/local/cuda/include:/usr/local/cuda/lib64 ${PREFIX:-}/include:${PREFIX:-}/lib ${CONDA_PREFIX:-}/include:${CONDA_PREFIX:-}/lib ${root_folder}/temp/cuda/include:${root_folder}/temp/cuda/lib64"
local gds_version=1.0.0
local candidate
local cufile_include
local cufile_lib
Expand All @@ -425,10 +426,10 @@ copy_gds_files_() {

local temp_tgz_dir=$(mktemp -d)
pushd ${temp_tgz_dir}
run_command wget https://developer.download.nvidia.com/gds/redist/rel-0.95.0/gds-redistrib-0.95.0.tgz
run_command tar xzvf gds-redistrib-0.95.0.tgz
run_command cp -P gds-redistrib-0.95.0/targets/x86_64-linux/include/cufile.h ${root_folder}/temp/cuda/include/
run_command cp -P gds-redistrib-0.95.0/targets/x86_64-linux/lib/* ${root_folder}/temp/cuda/lib64/
run_command wget https://developer.download.nvidia.com/gds/redist/rel-${gds_version}/gds-redistrib-${gds_version}.tgz
run_command tar xzvf gds-redistrib-${gds_version}.tgz
run_command cp -P gds-redistrib-${gds_version}/targets/x86_64-linux/include/cufile.h ${root_folder}/temp/cuda/include/
run_command cp -P gds-redistrib-${gds_version}/targets/x86_64-linux/lib/* ${root_folder}/temp/cuda/lib64/
popd > /dev/null
run_command rm -r ${temp_tgz_dir}
else
Expand Down