Enable GDS and Support Runtime Context (__enter__, __exit__) for CuFileDriver and CuImage

CMakeLists.txt

@@ -103,7 +103,7 @@ include(ExternalProject)
 ################################################################################
 # Options
 ################################################################################
-option(CUCIM_SUPPORT_GDS "Support cufile library" OFF)
+option(CUCIM_SUPPORT_GDS "Support cufile library" ON)
 option(CUCIM_STATIC_GDS "Use static cufile library" OFF)
 option(CUCIM_SUPPORT_CUDA "Support CUDA" ON)
 

cpp/CMakeLists.txt

@@ -65,6 +65,7 @@ add_library(${CUCIM_PACKAGE_NAME}
         include/cucim/memory/memory_manager.h
         include/cucim/util/cuda.h
         include/cucim/util/file.h
+        include/cucim/util/platform.h
         include/cucim/3rdparty/dlpack/dlpack.h
         include/cucim/3rdparty/dlpack/dlpackcpp.h
         src/cuimage.cpp
@@ -95,7 +96,8 @@ add_library(${CUCIM_PACKAGE_NAME}
         src/logger/logger.cpp
         src/logger/timer.cpp
         src/memory/memory_manager.cu
-        src/util/file.cpp)
+        src/util/file.cpp
+        src/util/platform.cpp)
 
 # Compile options
 set_target_properties(${CUCIM_PACKAGE_NAME}

cpp/include/cucim/cuimage.h

@@ -164,6 +164,8 @@ class EXPORT_VISIBLE CuImage : public std::enable_shared_from_this<CuImage>
 
     void save(std::string file_path) const;
 
+    void close();
+
 private:
     using Mutex = std::mutex;
     using ScopedLock = std::scoped_lock<Mutex>;

cpp/include/cucim/util/platform.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+//
+#ifndef CUCIM_UTIL_PLATFORM_H
+#define CUCIM_UTIL_PLATFORM_H
+
+#include "cucim/macros/api_header.h"
+
+/**
+ * @brief Platform-specific macros and functions.
+ */
+namespace cucim::util
+{
+
+EXPORT_VISIBLE bool is_in_wsl();
+
+} // namespace cucim::util
+
+#endif // CUCIM_UTIL_PLATFORM_H

cpp/src/cuimage.cpp

@@ -229,10 +229,7 @@ CuImage::CuImage() : std::enable_shared_from_this<CuImage>()
 CuImage::~CuImage()
 {
     //    printf("[cuCIM] CuImage::~CuImage()\n");
-    if (file_handle_.client_data)
-    {
-        image_formats_->formats[0].image_parser.close(&file_handle_);
-    }
+    close();
     image_formats_ = nullptr; // memory release is handled by the framework
     if (image_metadata_)
     {
@@ -615,6 +612,10 @@ CuImage CuImage::read_region(std::vector<int64_t>&& location,
         // Read region from internal file if image_data_ is nullptr
         if (image_data_ == nullptr)
         {
+            if (file_handle_.fd < 0) // file_handle_ is not opened
+            {
+                throw std::runtime_error("[Error] The image file is closed!");
+            }
             if (!image_formats_->formats[0].image_reader.read(
                     &file_handle_, image_metadata_, &request, image_data, nullptr /*out_metadata*/))
             {
@@ -778,6 +779,10 @@ std::set<std::string> CuImage::associated_images() const
 
 CuImage CuImage::associated_image(const std::string& name, const io::Device& device) const
 {
+    if (file_handle_.fd < 0) // file_handle_ is not opened
+    {
+        throw std::runtime_error("[Error] The image file is closed!");
+    }
     auto it = associated_images_.find(name);
     if (it != associated_images_.end())
     {
@@ -855,6 +860,18 @@ void CuImage::save(std::string file_path) const
         fs.close();
     }
 }
+
+void CuImage::close()
+{
+    if (file_handle_.client_data)
+    {
+        image_formats_->formats[0].image_parser.close(&file_handle_);
+    }
+    file_handle_.cufile = nullptr;
+    file_handle_.path = nullptr;
+    file_handle_.fd = -1;
+}
+
 void CuImage::ensure_init()
 {
     ScopedLock g(mutex_);

cpp/src/filesystem/cufile_driver.cpp

@@ -30,6 +30,7 @@
 #include <fmt/format.h>
 
 #include "cucim/util/cuda.h"
+#include "cucim/util/platform.h"
 #include "cufile_stub.h"
 
 #define ALIGN_UP(x, align_to) (((uint64_t)(x) + ((uint64_t)(align_to)-1)) & ~((uint64_t)(align_to)-1))
@@ -100,7 +101,7 @@ static int get_file_flags(const char* flags)
 
 bool is_gds_available()
 {
-    return static_cast<bool>(s_cufile_initializer);
+    return static_cast<bool>(s_cufile_initializer) && !cucim::util::is_in_wsl();
 }
 
 std::shared_ptr<CuFileDriver> open(const char* file_path, const char* flags, mode_t mode)
@@ -529,7 +530,8 @@ ssize_t CuFileDriver::pread(void* buf, size_t count, off_t file_offset, off_t bu
             {
                 if (memory_type == cudaMemoryTypeUnregistered)
                 {
-                    read_cnt = ::pread(handle_.fd, reinterpret_cast<char*>(buf) + buf_offset, block_read_size, file_offset);
+                    read_cnt =
+                        ::pread(handle_.fd, reinterpret_cast<char*>(buf) + buf_offset, block_read_size, file_offset);
                     total_read_cnt += read_cnt;
                 }
                 else
@@ -838,7 +840,8 @@ ssize_t CuFileDriver::pwrite(const void* buf, size_t count, off_t file_offset, o
             {
                 if (memory_type == cudaMemoryTypeUnregistered)
                 {
-                    write_cnt = ::pwrite(handle_.fd, reinterpret_cast<const char*>(buf) + buf_offset, block_write_size, file_offset);
+                    write_cnt = ::pwrite(
+                        handle_.fd, reinterpret_cast<const char*>(buf) + buf_offset, block_write_size, file_offset);
                     total_write_cnt += write_cnt;
                 }
                 else
@@ -1100,7 +1103,8 @@ ssize_t CuFileDriver::pwrite(const void* buf, size_t count, off_t file_offset, o
     {
         (void*)s_cufile_cache.device_cache(); // Lazy initialization
 
-        ssize_t write_cnt = cuFileWrite(handle_.cufile, reinterpret_cast<const char*>(buf) + buf_offset, count, file_offset, 0);
+        ssize_t write_cnt =
+            cuFileWrite(handle_.cufile, reinterpret_cast<const char*>(buf) + buf_offset, count, file_offset, 0);
         if (write_cnt < 0)
         {
             fmt::print(stderr, "[cuFile Error] {}\n", CUFILE_ERRSTR(write_cnt));

cpp/src/filesystem/file_handle.cpp

@@ -21,7 +21,7 @@
 #include "cucim/codec/hash_function.h"
 
 CuCIMFileHandle::CuCIMFileHandle()
-    : fd(0),
+    : fd(-1),
       cufile(nullptr),
       type(FileHandleType::kUnknown),
       path(nullptr),

cpp/src/util/file.cpp

@@ -1,3 +1,19 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 #include "cucim/util/file.h"
 
 #include <sys/stat.h>

cpp/src/util/platform.cpp

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cucim/util/platform.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <sys/utsname.h>
+
+
+namespace cucim::util
+{
+
+bool is_in_wsl()
+{
+    struct utsname buf;
+    int err = uname(&buf);
+    if (err == 0)
+    {
+        char* pos = strstr(buf.release, "icrosoft");
+        if (pos)
+        {
+            // 'Microsoft' for WSL1 and 'microsoft' for WSL2
+            if (buf.release < pos && (pos[-1] == 'm' || pos[-1] == 'M'))
+            {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+} // namespace cucim::util

gds/src/cufile_stub.cpp

@@ -16,6 +16,7 @@
 
 #include "cufile_stub.h"
 #include "cucim/dynlib/helper.h"
+#include "cucim/util/platform.h"
 
 #define IMPORT_FUNCTION(handle, name) impl_##name = cucim::dynlib::get_library_symbol<t_##name>(handle, #name);
 
@@ -196,9 +197,16 @@ extern "C"
 
     CUfileError_t cuFileDriverOpen(void)
     {
+        // GDS v1.0.0 does not support WSL and executing this can cause the following error:
+        //    Assertion failure, file index :cufio-udev  line :143
+        // So we do not call impl_cuFileDriverOpen() here if the current platform is WSL.
         if (impl_cuFileDriverOpen)
         {
-            return impl_cuFileDriverOpen();
+            // If not in WSL, call impl_cuFileDriverOpen()
+            if (!cucim::util::is_in_wsl())
+            {
+                return impl_cuFileDriverOpen();
+            }
         }
         return CUfileError_t{ CU_FILE_DRIVER_NOT_INITIALIZED, CUDA_SUCCESS };
     }

python/pybind11/cucim_py.cpp

@@ -153,7 +153,21 @@ PYBIND11_MODULE(_cucim, m)
              py::arg("name") = "", //
              py::arg("device") = io::Device()) //
         .def("save", &CuImage::save, doc::CuImage::doc_save, py::call_guard<py::gil_scoped_release>()) //
+        .def("close", &CuImage::close, doc::CuImage::doc_close, py::call_guard<py::gil_scoped_release>()) //
         .def("__bool__", &CuImage::operator bool, py::call_guard<py::gil_scoped_release>()) //
+        .def(
+            "__enter__",
+            [](const std::shared_ptr<CuImage>& cuimg) { //
+                return cuimg; //
+            }, //
+            py::call_guard<py::gil_scoped_release>())
+        .def(
+            "__exit__",
+            [](const std::shared_ptr<CuImage>& cuimg, const py::object& type, const py::object& value,
+               const py::object& traceback) { //
+                cuimg->close(); //
+            }, //
+            py::call_guard<py::gil_scoped_release>())
         .def(
             "__repr__", //
             [](const CuImage& cuimg) { //

python/pybind11/cucim_pydoc.h

@@ -227,6 +227,15 @@ Saves image data to the file path.
 Currently it supports only .ppm file format that can be viewed by `eog` command in Ubuntu.
 )doc")
 
+// void close();
+PYDOC(close, R"doc(
+Closes the file handle.
+
+Once the file handle is closed, the image object (if loaded before) still exists but cannot read additional images
+from the file.
+)doc")
+
+
 // void _set_array_interface(const CuImage& cuimg);
 PYDOC(_set_array_interface, R"doc(
 Add `__array_interface__` or `__cuda_array_interface__` depending on the memory type.

python/pybind11/filesystem/filesystem_py.cpp

@@ -58,6 +58,19 @@ void init_filesystem(py::module& fs)
              py::arg("file_offset"), //
              py::arg("buf_offset") = 0) //
         .def("close", &CuFileDriver::close, doc::CuFileDriver::doc_close, py::call_guard<py::gil_scoped_release>()) //
+        .def(
+            "__enter__",
+            [](const std::shared_ptr<CuFileDriver>& fd) { //
+                return fd; //
+            }, //
+            py::call_guard<py::gil_scoped_release>())
+        .def(
+            "__exit__",
+            [](const std::shared_ptr<CuFileDriver>& fd, const py::object& type, const py::object& value,
+               const py::object& traceback) { //
+                fd->close(); //
+            }, //
+            py::call_guard<py::gil_scoped_release>())
         .def("__repr__", [](const CuFileDriver& fd) {
             return fmt::format("<cucim.clara.filesystem.CuFileDriver path:{}>", fd.path());
         });

run

@@ -402,6 +402,7 @@ build_local() {
 copy_gds_files_() {
     local root_folder=${1:-${TOP}}
     local cufile_search="${root_folder}/temp/cuda/include:${root_folder}/temp/cuda/lib64"  #"/usr/local/cuda/include:/usr/local/cuda/lib64 ${PREFIX:-}/include:${PREFIX:-}/lib ${CONDA_PREFIX:-}/include:${CONDA_PREFIX:-}/lib ${root_folder}/temp/cuda/include:${root_folder}/temp/cuda/lib64"
+    local gds_version=1.0.0
     local candidate
     local cufile_include
     local cufile_lib
@@ -425,10 +426,10 @@ copy_gds_files_() {
 
         local temp_tgz_dir=$(mktemp -d)
         pushd ${temp_tgz_dir}
-        run_command wget https://developer.download.nvidia.com/gds/redist/rel-0.95.0/gds-redistrib-0.95.0.tgz
-        run_command tar xzvf gds-redistrib-0.95.0.tgz
-        run_command cp -P gds-redistrib-0.95.0/targets/x86_64-linux/include/cufile.h ${root_folder}/temp/cuda/include/
-        run_command cp -P gds-redistrib-0.95.0/targets/x86_64-linux/lib/* ${root_folder}/temp/cuda/lib64/
+        run_command wget https://developer.download.nvidia.com/gds/redist/rel-${gds_version}/gds-redistrib-${gds_version}.tgz
+        run_command tar xzvf gds-redistrib-${gds_version}.tgz
+        run_command cp -P gds-redistrib-${gds_version}/targets/x86_64-linux/include/cufile.h ${root_folder}/temp/cuda/include/
+        run_command cp -P gds-redistrib-${gds_version}/targets/x86_64-linux/lib/* ${root_folder}/temp/cuda/lib64/
         popd > /dev/null
         run_command rm -r ${temp_tgz_dir}
     else