From 7f84ed84e2e3c4344acd133762d8c5776e08020a Mon Sep 17 00:00:00 2001 From: Gigon Bae Date: Fri, 24 Sep 2021 17:57:03 -0700 Subject: [PATCH 1/4] Enable GDS with GDS v1.0.0 --- CMakeLists.txt | 2 +- run | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 861e6cd37..ee7701cbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -103,7 +103,7 @@ include(ExternalProject) ################################################################################ # Options ################################################################################ -option(CUCIM_SUPPORT_GDS "Support cufile library" OFF) +option(CUCIM_SUPPORT_GDS "Support cufile library" ON) option(CUCIM_STATIC_GDS "Use static cufile library" OFF) option(CUCIM_SUPPORT_CUDA "Support CUDA" ON) diff --git a/run b/run index 9ef6401e8..4be7b0190 100755 --- a/run +++ b/run @@ -402,6 +402,7 @@ build_local() { copy_gds_files_() { local root_folder=${1:-${TOP}} local cufile_search="${root_folder}/temp/cuda/include:${root_folder}/temp/cuda/lib64" #"/usr/local/cuda/include:/usr/local/cuda/lib64 ${PREFIX:-}/include:${PREFIX:-}/lib ${CONDA_PREFIX:-}/include:${CONDA_PREFIX:-}/lib ${root_folder}/temp/cuda/include:${root_folder}/temp/cuda/lib64" + local gds_version=1.0.0 local candidate local cufile_include local cufile_lib @@ -425,10 +426,10 @@ copy_gds_files_() { local temp_tgz_dir=$(mktemp -d) pushd ${temp_tgz_dir} - run_command wget https://developer.download.nvidia.com/gds/redist/rel-0.95.0/gds-redistrib-0.95.0.tgz - run_command tar xzvf gds-redistrib-0.95.0.tgz - run_command cp -P gds-redistrib-0.95.0/targets/x86_64-linux/include/cufile.h ${root_folder}/temp/cuda/include/ - run_command cp -P gds-redistrib-0.95.0/targets/x86_64-linux/lib/* ${root_folder}/temp/cuda/lib64/ + run_command wget https://developer.download.nvidia.com/gds/redist/rel-${gds_version}/gds-redistrib-${gds_version}.tgz + run_command tar xzvf gds-redistrib-${gds_version}.tgz + run_command cp -P gds-redistrib-${gds_version}/targets/x86_64-linux/include/cufile.h ${root_folder}/temp/cuda/include/ + run_command cp -P gds-redistrib-${gds_version}/targets/x86_64-linux/lib/* ${root_folder}/temp/cuda/lib64/ popd > /dev/null run_command rm -r ${temp_tgz_dir} else From d01cf1367dc2b7eefd30d016a409811d9631cf9d Mon Sep 17 00:00:00 2001 From: Gigon Bae Date: Mon, 27 Sep 2021 17:25:26 -0700 Subject: [PATCH 2/4] Do not use GDS if in WSL GDS v1.0.0 does not support WSL and executing this can cause the following error: Assertion failure, file index :cufio-udev line :143 So do not call cuFileDriverOpen() if the current platform is WSL. --- cpp/CMakeLists.txt | 4 ++- cpp/include/cucim/util/platform.h | 32 +++++++++++++++++++ cpp/src/filesystem/cufile_driver.cpp | 12 +++++--- cpp/src/util/file.cpp | 16 ++++++++++ cpp/src/util/platform.cpp | 46 ++++++++++++++++++++++++++++ gds/src/cufile_stub.cpp | 10 +++++- 6 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 cpp/include/cucim/util/platform.h create mode 100644 cpp/src/util/platform.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7bef1f7e3..804ee3368 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -65,6 +65,7 @@ add_library(${CUCIM_PACKAGE_NAME} include/cucim/memory/memory_manager.h include/cucim/util/cuda.h include/cucim/util/file.h + include/cucim/util/platform.h include/cucim/3rdparty/dlpack/dlpack.h include/cucim/3rdparty/dlpack/dlpackcpp.h src/cuimage.cpp @@ -95,7 +96,8 @@ add_library(${CUCIM_PACKAGE_NAME} src/logger/logger.cpp src/logger/timer.cpp src/memory/memory_manager.cu - src/util/file.cpp) + src/util/file.cpp + src/util/platform.cpp) # Compile options set_target_properties(${CUCIM_PACKAGE_NAME} diff --git a/cpp/include/cucim/util/platform.h b/cpp/include/cucim/util/platform.h new file mode 100644 index 000000000..699762756 --- /dev/null +++ b/cpp/include/cucim/util/platform.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// +#ifndef CUCIM_UTIL_PLATFORM_H +#define CUCIM_UTIL_PLATFORM_H + +#include "cucim/macros/api_header.h" + +/** + * @brief Platform-specific macros and functions. + */ +namespace cucim::util +{ + +EXPORT_VISIBLE bool is_in_wsl(); + +} // namespace cucim::util + +#endif // CUCIM_UTIL_PLATFORM_H diff --git a/cpp/src/filesystem/cufile_driver.cpp b/cpp/src/filesystem/cufile_driver.cpp index 23555f2ce..7f6993d3b 100644 --- a/cpp/src/filesystem/cufile_driver.cpp +++ b/cpp/src/filesystem/cufile_driver.cpp @@ -30,6 +30,7 @@ #include #include "cucim/util/cuda.h" +#include "cucim/util/platform.h" #include "cufile_stub.h" #define ALIGN_UP(x, align_to) (((uint64_t)(x) + ((uint64_t)(align_to)-1)) & ~((uint64_t)(align_to)-1)) @@ -100,7 +101,7 @@ static int get_file_flags(const char* flags) bool is_gds_available() { - return static_cast(s_cufile_initializer); + return static_cast(s_cufile_initializer) && !cucim::util::is_in_wsl(); } std::shared_ptr open(const char* file_path, const char* flags, mode_t mode) @@ -529,7 +530,8 @@ ssize_t CuFileDriver::pread(void* buf, size_t count, off_t file_offset, off_t bu { if (memory_type == cudaMemoryTypeUnregistered) { - read_cnt = ::pread(handle_.fd, reinterpret_cast(buf) + buf_offset, block_read_size, file_offset); + read_cnt = + ::pread(handle_.fd, reinterpret_cast(buf) + buf_offset, block_read_size, file_offset); total_read_cnt += read_cnt; } else @@ -838,7 +840,8 @@ ssize_t CuFileDriver::pwrite(const void* buf, size_t count, off_t file_offset, o { if (memory_type == cudaMemoryTypeUnregistered) { - write_cnt = ::pwrite(handle_.fd, reinterpret_cast(buf) + buf_offset, block_write_size, file_offset); + write_cnt = ::pwrite( + handle_.fd, reinterpret_cast(buf) + buf_offset, block_write_size, file_offset); total_write_cnt += write_cnt; } else @@ -1100,7 +1103,8 @@ ssize_t CuFileDriver::pwrite(const void* buf, size_t count, off_t file_offset, o { (void*)s_cufile_cache.device_cache(); // Lazy initialization - ssize_t write_cnt = cuFileWrite(handle_.cufile, reinterpret_cast(buf) + buf_offset, count, file_offset, 0); + ssize_t write_cnt = + cuFileWrite(handle_.cufile, reinterpret_cast(buf) + buf_offset, count, file_offset, 0); if (write_cnt < 0) { fmt::print(stderr, "[cuFile Error] {}\n", CUFILE_ERRSTR(write_cnt)); diff --git a/cpp/src/util/file.cpp b/cpp/src/util/file.cpp index 7009ae792..08d659372 100644 --- a/cpp/src/util/file.cpp +++ b/cpp/src/util/file.cpp @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "cucim/util/file.h" #include diff --git a/cpp/src/util/platform.cpp b/cpp/src/util/platform.cpp new file mode 100644 index 000000000..de0732633 --- /dev/null +++ b/cpp/src/util/platform.cpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cucim/util/platform.h" + +#include +#include +#include + + +namespace cucim::util +{ + +bool is_in_wsl() +{ + struct utsname buf; + int err = uname(&buf); + if (err == 0) + { + char* pos = strstr(buf.release, "icrosoft"); + if (pos) + { + // 'Microsoft' for WSL1 and 'microsoft' for WSL2 + if (buf.release < pos && (pos[-1] == 'm' || pos[-1] == 'M')) + { + return true; + } + } + } + return false; +} + +} // namespace cucim::util diff --git a/gds/src/cufile_stub.cpp b/gds/src/cufile_stub.cpp index 1cc4020a1..bf52e758b 100644 --- a/gds/src/cufile_stub.cpp +++ b/gds/src/cufile_stub.cpp @@ -16,6 +16,7 @@ #include "cufile_stub.h" #include "cucim/dynlib/helper.h" +#include "cucim/util/platform.h" #define IMPORT_FUNCTION(handle, name) impl_##name = cucim::dynlib::get_library_symbol(handle, #name); @@ -196,9 +197,16 @@ extern "C" CUfileError_t cuFileDriverOpen(void) { + // GDS v1.0.0 does not support WSL and executing this can cause the following error: + // Assertion failure, file index :cufio-udev line :143 + // So we do not call impl_cuFileDriverOpen() here if the current platform is WSL. if (impl_cuFileDriverOpen) { - return impl_cuFileDriverOpen(); + // If not in WSL, call impl_cuFileDriverOpen() + if (!cucim::util::is_in_wsl()) + { + return impl_cuFileDriverOpen(); + } } return CUfileError_t{ CU_FILE_DRIVER_NOT_INITIALIZED, CUDA_SUCCESS }; } From edb624d7fcc35858ae8f4f48ee7e85a12d3a883f Mon Sep 17 00:00:00 2001 From: Gigon Bae Date: Mon, 27 Sep 2021 18:05:36 -0700 Subject: [PATCH 3/4] Add __enter__ and __exit__ for CuFileDriver --- python/pybind11/filesystem/filesystem_py.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/pybind11/filesystem/filesystem_py.cpp b/python/pybind11/filesystem/filesystem_py.cpp index 40c9dc2bc..e11de2bc5 100644 --- a/python/pybind11/filesystem/filesystem_py.cpp +++ b/python/pybind11/filesystem/filesystem_py.cpp @@ -58,6 +58,19 @@ void init_filesystem(py::module& fs) py::arg("file_offset"), // py::arg("buf_offset") = 0) // .def("close", &CuFileDriver::close, doc::CuFileDriver::doc_close, py::call_guard()) // + .def( + "__enter__", + [](const std::shared_ptr& fd) { // + return fd; // + }, // + py::call_guard()) + .def( + "__exit__", + [](const std::shared_ptr& fd, const py::object& type, const py::object& value, + const py::object& traceback) { // + fd->close(); // + }, // + py::call_guard()) .def("__repr__", [](const CuFileDriver& fd) { return fmt::format("", fd.path()); }); From 2d80efe81eb744adee1d73a02365fd1288573086 Mon Sep 17 00:00:00 2001 From: Gigon Bae Date: Tue, 28 Sep 2021 11:27:53 -0700 Subject: [PATCH 4/4] Add __enter__ and __exit__ for CuImage --- cpp/include/cucim/cuimage.h | 2 ++ cpp/src/cuimage.cpp | 25 +++++++++++++++++++++---- cpp/src/filesystem/file_handle.cpp | 2 +- python/pybind11/cucim_py.cpp | 14 ++++++++++++++ python/pybind11/cucim_pydoc.h | 9 +++++++++ 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/cpp/include/cucim/cuimage.h b/cpp/include/cucim/cuimage.h index 9e4f0dcc1..18efc028b 100644 --- a/cpp/include/cucim/cuimage.h +++ b/cpp/include/cucim/cuimage.h @@ -164,6 +164,8 @@ class EXPORT_VISIBLE CuImage : public std::enable_shared_from_this void save(std::string file_path) const; + void close(); + private: using Mutex = std::mutex; using ScopedLock = std::scoped_lock; diff --git a/cpp/src/cuimage.cpp b/cpp/src/cuimage.cpp index 6f126deb5..8942c557e 100644 --- a/cpp/src/cuimage.cpp +++ b/cpp/src/cuimage.cpp @@ -229,10 +229,7 @@ CuImage::CuImage() : std::enable_shared_from_this() CuImage::~CuImage() { // printf("[cuCIM] CuImage::~CuImage()\n"); - if (file_handle_.client_data) - { - image_formats_->formats[0].image_parser.close(&file_handle_); - } + close(); image_formats_ = nullptr; // memory release is handled by the framework if (image_metadata_) { @@ -615,6 +612,10 @@ CuImage CuImage::read_region(std::vector&& location, // Read region from internal file if image_data_ is nullptr if (image_data_ == nullptr) { + if (file_handle_.fd < 0) // file_handle_ is not opened + { + throw std::runtime_error("[Error] The image file is closed!"); + } if (!image_formats_->formats[0].image_reader.read( &file_handle_, image_metadata_, &request, image_data, nullptr /*out_metadata*/)) { @@ -778,6 +779,10 @@ std::set CuImage::associated_images() const CuImage CuImage::associated_image(const std::string& name, const io::Device& device) const { + if (file_handle_.fd < 0) // file_handle_ is not opened + { + throw std::runtime_error("[Error] The image file is closed!"); + } auto it = associated_images_.find(name); if (it != associated_images_.end()) { @@ -855,6 +860,18 @@ void CuImage::save(std::string file_path) const fs.close(); } } + +void CuImage::close() +{ + if (file_handle_.client_data) + { + image_formats_->formats[0].image_parser.close(&file_handle_); + } + file_handle_.cufile = nullptr; + file_handle_.path = nullptr; + file_handle_.fd = -1; +} + void CuImage::ensure_init() { ScopedLock g(mutex_); diff --git a/cpp/src/filesystem/file_handle.cpp b/cpp/src/filesystem/file_handle.cpp index 1b3036d94..dfb55c8de 100644 --- a/cpp/src/filesystem/file_handle.cpp +++ b/cpp/src/filesystem/file_handle.cpp @@ -21,7 +21,7 @@ #include "cucim/codec/hash_function.h" CuCIMFileHandle::CuCIMFileHandle() - : fd(0), + : fd(-1), cufile(nullptr), type(FileHandleType::kUnknown), path(nullptr), diff --git a/python/pybind11/cucim_py.cpp b/python/pybind11/cucim_py.cpp index 65c140178..753cac9bc 100644 --- a/python/pybind11/cucim_py.cpp +++ b/python/pybind11/cucim_py.cpp @@ -153,7 +153,21 @@ PYBIND11_MODULE(_cucim, m) py::arg("name") = "", // py::arg("device") = io::Device()) // .def("save", &CuImage::save, doc::CuImage::doc_save, py::call_guard()) // + .def("close", &CuImage::close, doc::CuImage::doc_close, py::call_guard()) // .def("__bool__", &CuImage::operator bool, py::call_guard()) // + .def( + "__enter__", + [](const std::shared_ptr& cuimg) { // + return cuimg; // + }, // + py::call_guard()) + .def( + "__exit__", + [](const std::shared_ptr& cuimg, const py::object& type, const py::object& value, + const py::object& traceback) { // + cuimg->close(); // + }, // + py::call_guard()) .def( "__repr__", // [](const CuImage& cuimg) { // diff --git a/python/pybind11/cucim_pydoc.h b/python/pybind11/cucim_pydoc.h index 1fa4b7af6..8d985535f 100644 --- a/python/pybind11/cucim_pydoc.h +++ b/python/pybind11/cucim_pydoc.h @@ -227,6 +227,15 @@ Saves image data to the file path. Currently it supports only .ppm file format that can be viewed by `eog` command in Ubuntu. )doc") +// void close(); +PYDOC(close, R"doc( +Closes the file handle. + +Once the file handle is closed, the image object (if loaded before) still exists but cannot read additional images +from the file. +)doc") + + // void _set_array_interface(const CuImage& cuimg); PYDOC(_set_array_interface, R"doc( Add `__array_interface__` or `__cuda_array_interface__` depending on the memory type.